<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.0">


  <link rel="apple-touch-icon" sizes="180x180" href="/blog/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/blog/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/blog/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/blog/images/logo.svg" color="#222">

<link rel="stylesheet" href="/blog/css/main.css">



<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.2/css/all.min.css">
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@3.1.1/animate.min.css">

<script class="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"littlefxc.github.io","root":"/blog/","images":"/blog/images","scheme":"Mist","version":"8.2.2","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":false,"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果：${query}","hits_time":"找到 ${hits} 个搜索结果（用时 ${time} 毫秒）","hits":"找到 ${hits} 个搜索结果"},"path":"/blog/search.xml","localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false}};
  </script>
<meta name="description" content="[TOC] 1 Kafka 核心概念详解1.1 Kafka(MQ) 的应用场景1.1.1 Kafka(MQ)之异步化、服务解耦、削峰填谷 异步化   服务解耦、削峰填谷">
<meta property="og:type" content="article">
<meta property="og:title" content="Kafka核心概念">
<meta property="og:url" content="http://littlefxc.github.io/2021/02/02/Kafka/index.html">
<meta property="og:site_name" content="一年春又来">
<meta property="og:description" content="[TOC] 1 Kafka 核心概念详解1.1 Kafka(MQ) 的应用场景1.1.1 Kafka(MQ)之异步化、服务解耦、削峰填谷 异步化   服务解耦、削峰填谷">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_1.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_2.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_3.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210203211950113.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_4.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_5.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_6.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_7.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_8.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_9.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_10.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_11.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_12.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_13.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_14.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210225221350755.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/format,png.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/format,png-20210322221324553.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/format,png-20210322221608062.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210325223028295.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210325232006969.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210329224142774.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210329224552763.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210331223055144.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210331233706053.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406221942118.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406224758405.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406230608886.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408215126380.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408224353954.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408225148721.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408230441675.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210419220943341.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210420223154642.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110610852.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110631185.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210426224407408.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110656500.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110711964.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110725934.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/2020080611073643.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110746949.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110759401.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110810452.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110822997.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110834262.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/20200806110843888.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210426230029824.png">
<meta property="article:published_time" content="2021-02-02T15:24:14.000Z">
<meta property="article:modified_time" content="2021-04-27T13:40:07.283Z">
<meta property="article:author" content="一年春又来">
<meta property="article:tag" content="mq">
<meta property="article:tag" content="kafka">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_1.png">


<link rel="canonical" href="http://littlefxc.github.io/2021/02/02/Kafka/">


<script class="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>
<title>Kafka核心概念 | 一年春又来</title>
  




  <noscript>
  <style>
  body { margin-top: 2rem; }

  .use-motion .menu-item,
  .use-motion .sidebar,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header {
    visibility: visible;
  }

  .use-motion .header,
  .use-motion .site-brand-container .toggle,
  .use-motion .footer { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle,
  .use-motion .custom-logo-image {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line {
    transform: scaleX(1);
  }

  .search-pop-overlay, .sidebar-nav { display: none; }
  .sidebar-panel { display: block; }
  </style>
</noscript>

<link rel="alternate" href="/blog/atom.xml" title="一年春又来" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏" role="button">
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/blog/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">一年春又来</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>



<nav class="site-nav">
  <ul class="main-menu menu">
        <li class="menu-item menu-item-home"><a href="/blog/" rel="section"><i class="home                          //首页 fa-fw"></i>首页</a></li>
        <li class="menu-item menu-item-archives"><a href="/blog/archives/" rel="section"><i class="archive          //归档 fa-fw"></i>归档</a></li>
        <li class="menu-item menu-item-categories"><a href="/blog/categories/" rel="section"><i class="th           //分类 fa-fw"></i>分类</a></li>
        <li class="menu-item menu-item-tags"><a href="/blog/tags/" rel="section"><i class="tags                     //标签 fa-fw"></i>标签</a></li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup"><div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off" maxlength="80"
           placeholder="搜索..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close" role="button">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div class="search-result-container no-result">
  <div class="search-result-icon">
    <i class="fa fa-spinner fa-pulse fa-5x"></i>
  </div>
</div>

    </div>
  </div>

</div>
        
  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>

  <aside class="sidebar">

    <div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
            <div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#1-Kafka-%E6%A0%B8%E5%BF%83%E6%A6%82%E5%BF%B5%E8%AF%A6%E8%A7%A3"><span class="nav-text">1 Kafka 核心概念详解</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#1-1-Kafka-MQ-%E7%9A%84%E5%BA%94%E7%94%A8%E5%9C%BA%E6%99%AF"><span class="nav-text">1.1 Kafka(MQ) 的应用场景</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-1-1-Kafka-MQ-%E4%B9%8B%E5%BC%82%E6%AD%A5%E5%8C%96%E3%80%81%E6%9C%8D%E5%8A%A1%E8%A7%A3%E8%80%A6%E3%80%81%E5%89%8A%E5%B3%B0%E5%A1%AB%E8%B0%B7"><span class="nav-text">1.1.1 Kafka(MQ)之异步化、服务解耦、削峰填谷</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-1-2-Kafka-%E6%B5%B7%E9%87%8F%E6%97%A5%E5%BF%97%E6%94%B6%E9%9B%86"><span class="nav-text">1.1.2 Kafka 海量日志收集</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-2-Kafka-%E5%9F%BA%E6%9C%AC%E6%A6%82%E5%BF%B5"><span class="nav-text">1.2 Kafka 基本概念</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-2-1-%E9%9B%86%E7%BE%A4%E6%9E%B6%E6%9E%84%E6%A6%82%E5%BF%B5"><span class="nav-text">1.2.1 集群架构概念</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-2-2-Topic%E3%80%81Partition"><span class="nav-text">1.2.2 Topic、Partition</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-2-3-%E5%89%AF%E6%9C%AC-replica"><span class="nav-text">1.2.3 副本(replica)</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-2-4-ISR%E8%AF%A6%E8%A7%A3-In-Sync-Replicas"><span class="nav-text">1.2.4 ISR详解(In Sync Replicas)</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-3-Kafka-%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA"><span class="nav-text">1.3 Kafka 环境搭建</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-4-Kafka-%E6%9E%81%E9%80%9F%E5%85%A5%E9%97%A8"><span class="nav-text">1.4 Kafka 极速入门</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-4-1-%E6%9E%84%E5%BB%BA%E7%94%9F%E4%BA%A7%E8%80%85%E6%AD%A5%E9%AA%A4"><span class="nav-text">1.4.1 构建生产者步骤</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-4-2-%E6%9E%84%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85%E6%AD%A5%E9%AA%A4"><span class="nav-text">1.4.2 构建消费者步骤</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-4-3-%E4%BB%A3%E7%A0%81%E5%AE%9E%E7%8E%B0"><span class="nav-text">1.4.3 代码实现</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#1-4-3-1-%E9%85%8D%E7%BD%AE%E7%B1%BB"><span class="nav-text">1.4.3.1 配置类</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#1-4-3-2-%E7%94%9F%E4%BA%A7%E8%80%85"><span class="nav-text">1.4.3.2 生产者</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#1-4-3-3-%E6%B6%88%E8%B4%B9%E8%80%85"><span class="nav-text">1.4.3.3 消费者</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#1-4-3-4-%E6%8B%A6%E6%88%AA%E5%99%A8"><span class="nav-text">1.4.3.4 拦截器</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-5-Kafka-%E5%9F%BA%E6%9C%AC%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0%E8%AE%B2%E8%A7%A3"><span class="nav-text">1.5 Kafka 基本配置参数讲解</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-6-Kafka-%E4%B9%8B%E7%94%9F%E4%BA%A7%E8%80%85"><span class="nav-text">1.6 Kafka 之生产者</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-6-1-%E5%8F%91%E9%80%81%E6%B6%88%E6%81%AF%EF%BC%9AProducerRecord"><span class="nav-text">1.6.1 发送消息：ProducerRecord</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-6-2-%E5%BF%85%E8%A6%81%E7%9A%84%E5%8F%82%E6%95%B0%E9%85%8D%E7%BD%AE%E9%A1%B9"><span class="nav-text">1.6.2 必要的参数配置项</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-6-3-%E5%8F%91%E9%80%81%E6%B6%88%E6%81%AF%E7%9A%843%E7%A7%8D%E6%96%B9%E6%B3%95"><span class="nav-text">1.6.3 发送消息的3种方法</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-6-4-KafkaProducer-%E6%B6%88%E6%81%AF%E5%8F%91%E9%80%81%E9%87%8D%E8%AF%95%E6%9C%BA%E5%88%B6"><span class="nav-text">1.6.4 KafkaProducer 消息发送重试机制</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-7-Kafka-%E4%B9%8B%E7%94%9F%E4%BA%A7%E8%80%85%E9%87%8D%E8%A6%81%E5%8F%82%E6%95%B0%E8%AF%A6%E8%A7%A3"><span class="nav-text">1.7 Kafka 之生产者重要参数详解</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-8-Kafka-%E4%B9%8B%E6%8B%A6%E6%88%AA%E5%99%A8"><span class="nav-text">1.8 Kafka 之拦截器</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-9-Kafka-%E4%B9%8B%E5%BA%8F%E5%88%97%E5%8C%96%E5%92%8C%E5%8F%8D%E5%BA%8F%E5%88%97%E5%8C%96"><span class="nav-text">1.9 Kafka 之序列化和反序列化</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-10-Kafka-%E4%B9%8B%E5%88%86%E5%8C%BA%E5%99%A8"><span class="nav-text">1.10 Kafka 之分区器</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-11-Kafka-%E4%B9%8B%E6%B6%88%E8%B4%B9%E8%80%85%E4%B8%8E%E6%B6%88%E8%B4%B9%E8%80%85%E7%BB%84"><span class="nav-text">1.11 Kafka 之消费者与消费者组</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-1-%E6%A6%82%E5%BF%B5"><span class="nav-text">1.11.1 概念</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-2-%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6%E6%A8%A1%E5%9E%8B"><span class="nav-text">1.11.2 消息中间件模型</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-3-Kafka-%E6%B6%88%E8%B4%B9%E8%80%85%E5%BF%85%E8%A6%81%E5%8F%82%E6%95%B0%E6%96%B9%E6%B3%95"><span class="nav-text">1.11.3 Kafka 消费者必要参数方法</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-4-kafka-%E6%B6%88%E8%B4%B9%E8%80%85%E6%8F%90%E4%BA%A4%E4%BD%8D%E7%A7%BB"><span class="nav-text">1.11.4 kafka 消费者提交位移</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-5-%E6%B6%88%E8%B4%B9%E8%80%85subscribe-%E4%B8%8E-assign-%E8%AF%A6%E8%A7%A3"><span class="nav-text">1.11.5 消费者subscribe 与 assign 详解</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-6-Kafka%E6%B6%88%E8%B4%B9%E8%80%85%E4%B9%8B%E5%A4%9A%E7%BA%BF%E7%A8%8B"><span class="nav-text">1.11.6 Kafka消费者之多线程</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-11-7-Kafka-%E6%B6%88%E8%B4%B9%E8%80%85%E9%87%8D%E8%A6%81%E5%8F%82%E6%95%B0"><span class="nav-text">1.11.7 Kafka 消费者重要参数</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#1-12-Kafka-%E9%AB%98%E7%BA%A7%E5%BA%94%E7%94%A8%E6%95%B4%E5%90%88-Spring-Boot"><span class="nav-text">1.12 Kafka 高级应用整合 Spring Boot</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-1-%E6%A0%B8%E5%BF%83%E4%BE%9D%E8%B5%96"><span class="nav-text">1.12.1 核心依赖</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-2-%E7%94%9F%E4%BA%A7%E8%80%85-application-properties-%E9%85%8D%E7%BD%AE"><span class="nav-text">1.12.2 生产者 application.properties 配置</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-3-KafkaProducerService"><span class="nav-text">1.12.3 KafkaProducerService</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-4-%E6%B6%88%E8%B4%B9%E8%80%85-application-properties-%E9%85%8D%E7%BD%AE"><span class="nav-text">1.12.4 消费者 application.properties 配置</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-5-KafkaConsumerService"><span class="nav-text">1.12.5 KafkaConsumerService</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-12-6-%E5%8D%95%E5%85%83%E6%B5%8B%E8%AF%95"><span class="nav-text">1.12.6 单元测试</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#2-Kafka-%E6%B5%B7%E9%87%8F%E6%97%A5%E5%BF%97%E6%94%B6%E9%9B%86%E7%B3%BB%E7%BB%9F%E5%AE%9E%E6%88%98"><span class="nav-text">2 Kafka 海量日志收集系统实战</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#2-1-%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1"><span class="nav-text">2.1 架构设计</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2-2-%E6%97%A5%E5%BF%97%E8%BE%93%E5%87%BA"><span class="nav-text">2.2 日志输出</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2-3-%E6%97%A5%E5%BF%97%E6%94%B6%E9%9B%86"><span class="nav-text">2.3 日志收集</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2-4-%E6%97%A5%E5%BF%97%E8%BF%87%E6%BB%A4"><span class="nav-text">2.4 日志过滤</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2-5-%E6%97%A5%E5%BF%97%E6%8C%81%E4%B9%85%E5%8C%96%E3%80%81%E5%8F%AF%E8%A7%86%E5%8C%96"><span class="nav-text">2.5 日志持久化、可视化</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#3-Kafka-%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5"><span class="nav-text">3 Kafka 数据同步</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#3-1-%E4%BB%80%E4%B9%88%E6%98%AF%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%EF%BC%9F"><span class="nav-text">3.1 什么是数据同步？</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-2-%E5%A6%82%E4%BD%95%E5%8E%BB%E9%80%89%E6%8B%A9%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%E6%8A%80%E6%9C%AF%EF%BC%9F"><span class="nav-text">3.2 如何去选择数据同步技术？</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-1-%E5%9C%A8%E4%B8%9A%E5%8A%A1%E4%BB%A3%E7%A0%81%E4%B8%AD%E5%90%8C%E6%AD%A5"><span class="nav-text">3.2.1 在业务代码中同步</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-2-%E5%AE%9A%E6%97%B6%E4%BB%BB%E5%8A%A1%E5%90%8C%E6%AD%A5"><span class="nav-text">3.2.2 定时任务同步</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-3-%E9%80%9A%E8%BF%87MQ%E5%AE%9E%E7%8E%B0%E5%90%8C%E6%AD%A5"><span class="nav-text">3.2.3 通过MQ实现同步</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-4-%E9%80%9A%E8%BF%87Canal%E5%AE%9E%E7%8E%B0%E5%AE%9E%E6%97%B6%E5%90%8C%E6%AD%A5"><span class="nav-text">3.2.4 通过Canal实现实时同步</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-3-%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%E6%A1%86%E6%9E%B6%EF%BC%9ACanal"><span class="nav-text">3.3 数据同步框架：Canal</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-1-Canal%E7%AE%80%E4%BB%8B"><span class="nav-text">3.3.1 Canal简介</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-2-Canal%E5%B7%A5%E4%BD%9C%E5%8E%9F%E7%90%86"><span class="nav-text">3.3.2 Canal工作原理</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-3-Canal%E5%86%85%E9%83%A8%E5%8E%9F%E7%90%86"><span class="nav-text">3.3.3 Canal内部原理</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-4-Canal%E5%86%85%E9%83%A8%E7%BB%93%E6%9E%84"><span class="nav-text">3.3.4 Canal内部结构</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-5-Canal-%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87"><span class="nav-text">3.3.5 Canal 环境准备</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-5-1-%E8%AE%BE%E7%BD%AEMySQL%E8%BF%9C%E7%A8%8B%E8%AE%BF%E9%97%AE"><span class="nav-text">3.3.5.1 设置MySQL远程访问</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-5-2-MySQL%E9%85%8D%E7%BD%AE"><span class="nav-text">3.3.5.2 MySQL配置</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-5-3-MySQL%E5%88%9B%E5%BB%BA%E7%94%A8%E6%88%B7%E6%8E%88%E6%9D%83"><span class="nav-text">3.3.5.3 MySQL创建用户授权</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-6-Canal%E9%83%A8%E7%BD%B2%E5%AE%89%E8%A3%85"><span class="nav-text">3.3.6 Canal部署安装</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-6-1-%E4%B8%8B%E8%BD%BDCanal"><span class="nav-text">3.3.6.1 下载Canal</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-6-2-%E4%B8%8A%E4%BC%A0%E8%A7%A3%E5%8E%8B"><span class="nav-text">3.3.6.2 上传解压</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-6-3-%E4%BF%AE%E6%94%B9%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="nav-text">3.3.6.3 修改配置文件</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-7-%E5%90%AF%E5%8A%A8Canal"><span class="nav-text">3.3.7 启动Canal</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-8-%E6%B5%8B%E8%AF%95Canal"><span class="nav-text">3.3.8 测试Canal</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-9-%E5%AF%BC%E5%85%A5%E5%B9%B6%E4%BF%AE%E6%94%B9%E6%BA%90%E7%A0%81"><span class="nav-text">3.3.9 导入并修改源码</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-10-%E6%B5%8B%E8%AF%95%E6%95%B0%E6%8D%AE%E5%8F%98%E6%9B%B4"><span class="nav-text">3.3.10 测试数据变更</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-11-%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%E5%AE%9E%E7%8E%B0"><span class="nav-text">3.3.11 数据同步实现</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-11-1-%E9%9C%80%E6%B1%82"><span class="nav-text">3.3.11.1 需求</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#3-3-11-2-%E5%85%B7%E4%BD%93%E5%AE%9E%E7%8E%B0"><span class="nav-text">3.3.11.2 具体实现</span></a></li></ol></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#4-%E5%8F%82%E8%80%83%E8%B5%84%E6%BA%90"><span class="nav-text">4 参考资源</span></a></li></ol></div>
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author site-overview-item animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">一年春又来</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap site-overview-item animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/blog/archives/">
        
          <span class="site-state-item-count">184</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/blog/categories/">
          
        <span class="site-state-item-count">35</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/blog/tags/">
          
        <span class="site-state-item-count">115</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>



        </div>
      </div>
    </div>
  </aside>
  <div class="sidebar-dimmer"></div>


    </header>

    
  <div class="back-to-top" role="button">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


    <div class="main-inner post posts-expand">


  


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://littlefxc.github.io/2021/02/02/Kafka/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/blog/images/avatar.gif">
      <meta itemprop="name" content="一年春又来">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="一年春又来">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Kafka核心概念
        </h1>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-02-02 23:24:14" itemprop="dateCreated datePublished" datetime="2021-02-02T23:24:14+08:00">2021-02-02</time>
    </span>
      <span class="post-meta-item">
        <span class="post-meta-item-icon">
          <i class="far fa-calendar-check"></i>
        </span>
        <span class="post-meta-item-text">更新于</span>
        <time title="修改时间：2021-04-27 21:40:07" itemprop="dateModified" datetime="2021-04-27T21:40:07+08:00">2021-04-27</time>
      </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/blog/categories/mq/" itemprop="url" rel="index"><span itemprop="name">mq</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
        <p>[TOC]</p>
<h1 id="1-Kafka-核心概念详解"><a href="#1-Kafka-核心概念详解" class="headerlink" title="1 Kafka 核心概念详解"></a>1 Kafka 核心概念详解</h1><h2 id="1-1-Kafka-MQ-的应用场景"><a href="#1-1-Kafka-MQ-的应用场景" class="headerlink" title="1.1 Kafka(MQ) 的应用场景"></a>1.1 Kafka(MQ) 的应用场景</h2><h3 id="1-1-1-Kafka-MQ-之异步化、服务解耦、削峰填谷"><a href="#1-1-1-Kafka-MQ-之异步化、服务解耦、削峰填谷" class="headerlink" title="1.1.1 Kafka(MQ)之异步化、服务解耦、削峰填谷"></a>1.1.1 Kafka(MQ)之异步化、服务解耦、削峰填谷</h3><ul>
<li><p>异步化</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_1.png" alt="kafka_1_1"></p>
</li>
<li><p>服务解耦、削峰填谷</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_2.png" alt="kafka_1_2"></p>
</li>
</ul>
<span id="more"></span>

<h3 id="1-1-2-Kafka-海量日志收集"><a href="#1-1-2-Kafka-海量日志收集" class="headerlink" title="1.1.2 Kafka 海量日志收集"></a>1.1.2 Kafka 海量日志收集</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_3.png" alt="kafka_1_3"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210203211950113.png" alt="image-20210203211950113"></p>
<ul>
<li><p>Kafka 之数据同步应用</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_4.png" alt="kafka_1_4"></p>
</li>
<li><p>Kafka 之实时计算</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_5.png" alt="kafka_1_5"></p>
</li>
</ul>
<h2 id="1-2-Kafka-基本概念"><a href="#1-2-Kafka-基本概念" class="headerlink" title="1.2 Kafka 基本概念"></a>1.2 Kafka 基本概念</h2><h3 id="1-2-1-集群架构概念"><a href="#1-2-1-集群架构概念" class="headerlink" title="1.2.1 集群架构概念"></a>1.2.1 集群架构概念</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_6.png" alt="kafka_1_6"></p>
<h3 id="1-2-2-Topic、Partition"><a href="#1-2-2-Topic、Partition" class="headerlink" title="1.2.2 Topic、Partition"></a>1.2.2 Topic、Partition</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_7.png" alt="kafka_1_7"></p>
<h3 id="1-2-3-副本-replica"><a href="#1-2-3-副本-replica" class="headerlink" title="1.2.3 副本(replica)"></a>1.2.3 副本(replica)</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_8.png" alt="kafka_1_8"></p>
<h3 id="1-2-4-ISR详解-In-Sync-Replicas"><a href="#1-2-4-ISR详解-In-Sync-Replicas" class="headerlink" title="1.2.4 ISR详解(In Sync Replicas)"></a>1.2.4 ISR详解(In Sync Replicas)</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_9.png" alt="kafka_1_9"></p>
<p>上图表示拉取及时的情况</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_10.png" alt="kafka_1_10"></p>
<p>上图表示拉取滞后的情况。</p>
<p><font color=red><strong>PS: 当Kafka集群中的 leader 挂了之后，Kafka集群会重新选举leader，这是只有在 ISR 集合里面的Kafka才会被选举成为leader。</strong></font></p>
<ul>
<li><p>HW: High Watermark， 高水位线，消费者只能最多拉取到高水位线的消息</p>
</li>
<li><p>LEO: Log End Offset，日志文件的最后一条记录的 offset(偏移量)</p>
</li>
<li><p>ISR 集合与 HW 和 LEO 直接存在着密不可分的关系</p>
</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_11.png" alt="kafka_1_11"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_12.png" alt="kafka_1_12"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_13.png" alt="kafka_1_13"></p>
<p>上图右边的图形表示数据传入到leader节点，但还没有同步到follower节点上</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/kafka_1_14.png" alt="kafka_1_14"></p>
<p>上图HW移动了一格，表示 follower1 节点和follower2 节点都同步了第3条数据，而第4条数据因为follower2节点没有同步到，Kafka消费者就消费不了第4条数据。</p>
<h2 id="1-3-Kafka-环境搭建"><a href="#1-3-Kafka-环境搭建" class="headerlink" title="1.3 Kafka 环境搭建"></a>1.3 Kafka 环境搭建</h2><p><a target="_blank" rel="noopener" href="https://blog.csdn.net/Little_fxc/article/details/108626224?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522161236585916780269848148%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fblog.%2522%257D&request_id=161236585916780269848148&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~blog~first_rank_v1~rank_blog_v1-1-108626224.pc_v1_rank_blog_v1&utm_term=zookeeper&spm=1018.2226.3001.4450">zookeeper 集群搭建</a></p>
<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/luzhanshi/p/13369834.html">Kafka 集群搭建</a></p>
<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/pzb-shadow/p/13030365.html">Kafka Manager - Kafka集群管理工具</a><br><a target="_blank" rel="noopener" href="https://blog.csdn.net/asd136912/article/details/103735037">kafka 命令行工具常用命令行操作</a></p>
<h2 id="1-4-Kafka-极速入门"><a href="#1-4-Kafka-极速入门" class="headerlink" title="1.4 Kafka 极速入门"></a>1.4 Kafka 极速入门</h2><h3 id="1-4-1-构建生产者步骤"><a href="#1-4-1-构建生产者步骤" class="headerlink" title="1.4.1 构建生产者步骤"></a>1.4.1 构建生产者步骤</h3><ol>
<li>配置生产者参数属性和创建生产者对象</li>
<li>构建消息：ProducerRecord</li>
<li>发送消息</li>
<li>关闭生产者</li>
</ol>
<h3 id="1-4-2-构建消费者步骤"><a href="#1-4-2-构建消费者步骤" class="headerlink" title="1.4.2 构建消费者步骤"></a>1.4.2 构建消费者步骤</h3><ol>
<li>配置消费者参数属性和创建消费者对象</li>
<li>订阅主题</li>
<li>拉取消息并进行消费处理</li>
<li>提交消费偏移量，关闭消费者</li>
</ol>
<h3 id="1-4-3-代码实现"><a href="#1-4-3-代码实现" class="headerlink" title="1.4.3 代码实现"></a>1.4.3 代码实现</h3><h4 id="1-4-3-1-配置类"><a href="#1-4-3-1-配置类" class="headerlink" title="1.4.3.1 配置类"></a>1.4.3.1 配置类</h4><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * kafka配置类</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="meta">@Configuration</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">KafkaConfig</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Bean</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> KafkaProducer&lt;String, String&gt; <span class="title">producerRecord</span><span class="params">()</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">        Properties properties = <span class="keyword">new</span> Properties();</span><br><span class="line">        <span class="comment">// 配置kafka集群地址，不用将全部机器都写上，zk会自动发现全部的kafka broke</span></span><br><span class="line">        properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, <span class="string">&quot;localhost:9092,localhost:9093&quot;</span>);</span><br><span class="line">        <span class="comment">// 设置发送消息的应答方式</span></span><br><span class="line">        properties.setProperty(ProducerConfig.ACKS_CONFIG, <span class="string">&quot;all&quot;</span>);</span><br><span class="line">        <span class="comment">// 重试次数</span></span><br><span class="line">        properties.setProperty(ProducerConfig.RETRIES_CONFIG, <span class="string">&quot;3&quot;</span>);</span><br><span class="line">        <span class="comment">// 重试间隔时间</span></span><br><span class="line">        properties.setProperty(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, <span class="string">&quot;100&quot;</span>);</span><br><span class="line">        <span class="comment">// 一批次发送的消息大小 16KB</span></span><br><span class="line">        properties.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, <span class="string">&quot;16348&quot;</span>);</span><br><span class="line">        <span class="comment">// 一个批次等待时间,10ms</span></span><br><span class="line">        properties.setProperty(ProducerConfig.LINGER_MS_CONFIG, <span class="string">&quot;10&quot;</span>);</span><br><span class="line">        <span class="comment">// RecordAccumulator 缓冲区大小  32M，如果缓冲区满了会阻塞发送端</span></span><br><span class="line">        properties.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, <span class="string">&quot;33554432&quot;</span>);</span><br><span class="line">        <span class="comment">// 配置拦截器, 多个逗号隔开</span></span><br><span class="line">        properties.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, <span class="string">&quot;com.xiaolyuh.interceptor.TraceInterceptor&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Serializer&lt;String&gt; keySerializer = <span class="keyword">new</span> StringSerializer();</span><br><span class="line">        Serializer&lt;String&gt; valueSerializer = <span class="keyword">new</span> StringSerializer();</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> KafkaProducer&lt;&gt;(properties, keySerializer, valueSerializer);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h4 id="1-4-3-2-生产者"><a href="#1-4-3-2-生产者" class="headerlink" title="1.4.3.2 生产者"></a>1.4.3.2 生产者</h4><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@RunWith(SpringRunner.class)</span></span><br><span class="line"><span class="meta">@SpringBootTest</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">SpringBootStudentKafkaApplicationTests</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Autowired</span></span><br><span class="line">    <span class="keyword">private</span> KafkaProducer&lt;String, String&gt; kafkaProducer;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testSyncKafkaSend</span><span class="params">()</span> <span class="keyword">throws</span> Exception </span>&#123;</span><br><span class="line">        <span class="comment">// 同步发送测试</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = <span class="number">0</span>; i &lt; <span class="number">100</span>; i++) &#123;</span><br><span class="line">            ProducerRecord&lt;String, String&gt; producerRecord = <span class="keyword">new</span> ProducerRecord&lt;&gt;(<span class="string">&quot;test_cluster_topic&quot;</span>, <span class="string">&quot;key-&quot;</span> + i, <span class="string">&quot;value-&quot;</span> + i);</span><br><span class="line">            <span class="comment">// 同步发送，这里我们还可以指定发送到那个分区，还可以添加header</span></span><br><span class="line">            kafkaProducer.send(producerRecord, <span class="keyword">new</span> KafkaCallback&lt;&gt;(producerRecord)).get(<span class="number">50</span>, TimeUnit.MINUTES);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        System.out.println(<span class="string">&quot;ThreadName::&quot;</span> + Thread.currentThread().getName());</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testAsyncKafkaSend</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="comment">// 异步发送测试</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = <span class="number">0</span>; i &lt; <span class="number">100</span>; i++) &#123;</span><br><span class="line">            ProducerRecord&lt;String, String&gt; producerRecord = <span class="keyword">new</span> ProducerRecord&lt;&gt;(<span class="string">&quot;test_cluster_topic2&quot;</span>, <span class="string">&quot;key-&quot;</span> + i, <span class="string">&quot;value-&quot;</span> + i);</span><br><span class="line">            <span class="comment">// 异步发送，这里我们还可以指定发送到那个分区，还可以添加header</span></span><br><span class="line">            kafkaProducer.send(producerRecord, <span class="keyword">new</span> KafkaCallback&lt;&gt;(producerRecord));</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        System.out.println(<span class="string">&quot;ThreadName::&quot;</span> + Thread.currentThread().getName());</span><br><span class="line">        <span class="comment">// 记得刷新，否则消息有可能没有发出去</span></span><br><span class="line">        kafkaProducer.flush();</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 异步回调函数，该方法会在 Producer 收到 ack 时调用，当Exception不为空表示发送消息失败。</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * <span class="doctag">@param</span> &lt;K&gt;</span></span><br><span class="line"><span class="comment"> * <span class="doctag">@param</span> &lt;V&gt;</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">KafkaCallback</span>&lt;<span class="title">K</span>, <span class="title">V</span>&gt; <span class="keyword">implements</span> <span class="title">Callback</span> </span>&#123;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> ProducerRecord&lt;K, V&gt; producerRecord;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="title">KafkaCallback</span><span class="params">(ProducerRecord&lt;K, V&gt; producerRecord)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.producerRecord = producerRecord;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">onCompletion</span><span class="params">(RecordMetadata metadata, Exception exception)</span> </span>&#123;</span><br><span class="line">        System.out.println(<span class="string">&quot;ThreadName::&quot;</span> + Thread.currentThread().getName());</span><br><span class="line">        <span class="keyword">if</span> (Objects.isNull(exception)) &#123;</span><br><span class="line">            System.out.println(metadata.partition() + <span class="string">&quot;-&quot;</span> + metadata.offset() + <span class="string">&quot;:::&quot;</span> + producerRecord.key() + <span class="string">&quot;=&quot;</span> + producerRecord.value());</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> (Objects.nonNull(exception)) &#123;</span><br><span class="line">            <span class="comment">// TODO  告警，消息落库从发</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h4 id="1-4-3-3-消费者"><a href="#1-4-3-3-消费者" class="headerlink" title="1.4.3.3 消费者"></a>1.4.3.3 消费者</h4><p>Kafka中的消息消费是一个不断轮询的过程，消费者所要做的就是重复地调用<code>poll()</code>方法，而<code>poll()</code>方法返回的是所订阅的主题（分区）上的一组消息。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Component</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">KafkaConsumerDemo</span> </span>&#123;</span><br><span class="line">    ThreadPoolExecutor executor = <span class="keyword">new</span> ThreadPoolExecutor(<span class="number">1</span>, <span class="number">10</span>,</span><br><span class="line">            <span class="number">0L</span>, TimeUnit.MILLISECONDS, <span class="keyword">new</span> LinkedBlockingQueue&lt;Runnable&gt;(<span class="number">1</span>));</span><br><span class="line"></span><br><span class="line">    <span class="meta">@PostConstruct</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">startConsumer</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        executor.submit(() -&gt; &#123;</span><br><span class="line">            Properties properties = <span class="keyword">new</span> Properties();</span><br><span class="line">            properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, <span class="string">&quot;localhost:9092,localhost:9093&quot;</span>);</span><br><span class="line">            </span><br><span class="line">            <span class="comment">// 非常重要的属性配置：与我们的消费者订阅组有关系</span></span><br><span class="line">            properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, <span class="string">&quot;groupId&quot;</span>);</span><br><span class="line">            <span class="comment">// 消费者提交 offset：自动提交 &amp; 手工提交，默认是自动提交</span></span><br><span class="line">            properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, <span class="string">&quot;false&quot;</span>);</span><br><span class="line">            <span class="comment">// 请求超时时间</span></span><br><span class="line">            properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, <span class="string">&quot;60000&quot;</span>);</span><br><span class="line">            <span class="comment">// 序列化</span></span><br><span class="line">            Deserializer&lt;String&gt; keyDeserializer = <span class="keyword">new</span> StringDeserializer();</span><br><span class="line">            Deserializer&lt;String&gt; valueDeserializer = <span class="keyword">new</span> StringDeserializer();</span><br><span class="line">            <span class="comment">// 创建消费者对象</span></span><br><span class="line">            KafkaConsumer&lt;String, String&gt; consumer = <span class="keyword">new</span> KafkaConsumer&lt;&gt;(properties, keyDeserializer, valueDeserializer);</span><br><span class="line">            <span class="comment">// 订阅感兴趣的主题</span></span><br><span class="line">            consumer.subscribe(Arrays.asList(<span class="string">&quot;test_cluster_topic&quot;</span>));</span><br><span class="line"></span><br><span class="line">            <span class="comment">// KafkaConsumer的assignment（）方法来判定是否分配到了相应的分区，如果为空表示没有分配到分区</span></span><br><span class="line">            Set&lt;TopicPartition&gt; assignment = consumer.assignment();</span><br><span class="line">            <span class="keyword">while</span> (assignment.isEmpty()) &#123;</span><br><span class="line">                <span class="comment">// 阻塞1秒</span></span><br><span class="line">                consumer.poll(<span class="number">1000</span>);</span><br><span class="line">                assignment = consumer.assignment();</span><br><span class="line">            &#125;</span><br><span class="line"></span><br><span class="line">            <span class="comment">// KafkaConsumer 分配到了分区，开始消费</span></span><br><span class="line">            <span class="keyword">while</span> (<span class="keyword">true</span>) &#123;</span><br><span class="line">                <span class="comment">// 拉取记录，如果没有记录则柱塞1000ms。</span></span><br><span class="line">                ConsumerRecords&lt;String, String&gt; records = consumer.poll(<span class="number">1000</span>);</span><br><span class="line">                <span class="keyword">for</span> (ConsumerRecord&lt;String, String&gt; record : records) &#123;</span><br><span class="line">                    String traceId = <span class="keyword">new</span> String(record.headers().lastHeader(<span class="string">&quot;traceId&quot;</span>).value());</span><br><span class="line">                    System.out.printf(<span class="string">&quot;traceId = %s, offset = %d, key = %s, value = %s%n&quot;</span>, traceId, record.offset(), record.key(), record.value());</span><br><span class="line">                &#125;</span><br><span class="line"></span><br><span class="line">                <span class="comment">// 异步确认提交</span></span><br><span class="line">                consumer.commitAsync((offsets, exception) -&gt; &#123;</span><br><span class="line">                    <span class="keyword">if</span> (Objects.isNull(exception)) &#123;</span><br><span class="line">                        <span class="comment">// TODO 告警、落盘、重试</span></span><br><span class="line">                    &#125;</span><br><span class="line">                &#125;);</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;);</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h4 id="1-4-3-4-拦截器"><a href="#1-4-3-4-拦截器" class="headerlink" title="1.4.3.4 拦截器"></a>1.4.3.4 拦截器</h4><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 链路ID</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">TraceInterceptor</span> <span class="keyword">implements</span> <span class="title">ProducerInterceptor</span>&lt;<span class="title">String</span>, <span class="title">String</span>&gt; </span>&#123;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">int</span> errorCounter = <span class="number">0</span>;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">int</span> successCounter = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * 最先调用，读取配置信息，只调用一次</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Map&lt;String, ?&gt; configs)</span> </span>&#123;</span><br><span class="line">        System.out.println(JSON.toJSONString(configs));</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * 它运行在用户主线程中，在消息序列化和计算分区之前调用，这里最好不小修改topic 和分区参数，否则会出一些奇怪的现象。</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> record</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@return</span></span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> ProducerRecord&lt;String, String&gt; <span class="title">onSend</span><span class="params">(ProducerRecord&lt;String, String&gt; record)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">        Headers headers = <span class="keyword">new</span> RecordHeaders();</span><br><span class="line">        headers.add(<span class="string">&quot;traceId&quot;</span>, UUID.randomUUID().toString().getBytes(Charset.forName(<span class="string">&quot;UTF8&quot;</span>)));</span><br><span class="line">        <span class="comment">// 修改消息</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> ProducerRecord&lt;&gt;(record.topic(), record.partition(), record.key(), record.value(), headers);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * 该方法会在消息从 RecordAccumulator 成功发送到 Kafka Broker 之后，或者在发送过程 中失败时调用。</span></span><br><span class="line"><span class="comment">     * 并且通常都是在 producer 回调逻辑触发之前调用。</span></span><br><span class="line"><span class="comment">     * onAcknowledgement 运行在 producer 的 IO 线程中，因此不要在该方法中放入很重的逻辑，否则会拖慢 producer 的消息 发送效率。</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> metadata</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> exception</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">onAcknowledgement</span><span class="params">(RecordMetadata metadata, Exception exception)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">if</span> (Objects.isNull(exception)) &#123;</span><br><span class="line">            <span class="comment">// TODO  出错了</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * 关闭 interceptor，主要用于执行一些资源清理工作，只调用一次</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        System.out.println(<span class="string">&quot;==========close============&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h2 id="1-5-Kafka-基本配置参数讲解"><a href="#1-5-Kafka-基本配置参数讲解" class="headerlink" title="1.5 Kafka 基本配置参数讲解"></a>1.5 Kafka 基本配置参数讲解</h2><ul>
<li><p>配置文件: <code>$KAFKA_HOME/config/server.properties</code></p>
<ul>
<li><p>zookeeper.connect</p>
<p>CS格式参数，可以指定值为zk1:2181,zk2:2181,zk3:2181，不同Kafka集群可以指定：zk1:2181,zk2:2181,zk3:2181/kafka1，chroot只需要写一次</p>
</li>
<li><p>listeners</p>
<p>设置内网访问Kafka服务的监听器</p>
</li>
<li><p>broker.id</p>
<p>每个broker都可以用一个唯一的非负整数id进行标识；这个id可以作为broker的“名字”，并且它的存在使得broker无须混淆consumers就可以迁移到不同的host/port上。你可以选择任意你喜欢的数字作为id，只要id是唯一的即可。</p>
</li>
<li><p>log.dir 和 log.dirs</p>
<p>kafka存放数据的路径。这个路径并不是唯一的，可以是多个，路径之间只需要使用逗号分隔即可；每当创建新partition时，都会选择在包含最少partitions的路径下进行。</p>
</li>
<li><p>message.max.bytes</p>
<p>server可以接收的消息最大尺寸。重要的是，consumer和producer有关这个属性的设置必须同步，否则producer发布的消息对consumer来说太大。</p>
</li>
</ul>
</li>
<li><p><a target="_blank" rel="noopener" href="https://littlefxc.gitee.io/blog/passages/KAFKA-%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0%E8%AF%A6%E8%A7%A3/">详细配置参数</a></p>
</li>
</ul>
<h2 id="1-6-Kafka-之生产者"><a href="#1-6-Kafka-之生产者" class="headerlink" title="1.6 Kafka 之生产者"></a>1.6 Kafka 之生产者</h2><h3 id="1-6-1-发送消息：ProducerRecord"><a href="#1-6-1-发送消息：ProducerRecord" class="headerlink" title="1.6.1 发送消息：ProducerRecord"></a>1.6.1 发送消息：ProducerRecord</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">ProducerRecord</span>&lt;<span class="title">K</span>, <span class="title">V</span>&gt; </span>&#123;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> String topic;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Integer partition;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Headers headers;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> K key;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> V value;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Long timestamp;</span><br><span class="line">    <span class="comment">// ...</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>PS: 一条消息会通过 Key 去计算出来实际的 partition，按照 partitiion 去存储的。</p>
<h3 id="1-6-2-必要的参数配置项"><a href="#1-6-2-必要的参数配置项" class="headerlink" title="1.6.2 必要的参数配置项"></a>1.6.2 必要的参数配置项</h3><ul>
<li><p>bootstrap.servers：逗号分隔符，多个地址，防止单点故障</p>
</li>
<li><p>key.serializer, value.serializer：kafka实际发送的是二进制的内容，所以必须序列化</p>
</li>
<li><p>client.id：kafka 对应生产者的ID。如果不设置，Kafka 内部会自动生成一个非空字符串</p>
</li>
<li><p>简化的配置Key: ProducerConfig</p>
</li>
<li><p>KafkaProducer 是线程安全的（kafka消费者不是线程安全的）</p>
</li>
</ul>
<h3 id="1-6-3-发送消息的3种方法"><a href="#1-6-3-发送消息的3种方法" class="headerlink" title="1.6.3 发送消息的3种方法"></a>1.6.3 发送消息的3种方法</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210225221350755.png" alt="image-20210225221350755"></p>
<p>Kafka 发送消息提供了 3 种方法：</p>
<ul>
<li>sendOffsetsToTransaction: 事务相关</li>
<li>send(ProducerRecord&lt;K,V&gt;):Future<RecordMetadata>：异步，但是使用 Future.get()方法相当于同步</li>
<li>send(ProducerRecord&lt;K,V&gt;, Callback):Future<RecordMetadata>：异步，返回值会放到 Callback 回调函数里</li>
</ul>
<h3 id="1-6-4-KafkaProducer-消息发送重试机制"><a href="#1-6-4-KafkaProducer-消息发送重试机制" class="headerlink" title="1.6.4 KafkaProducer 消息发送重试机制"></a>1.6.4 KafkaProducer 消息发送重试机制</h3><ul>
<li>retries 参数</li>
<li>可重试异常(例如：网络抖动) &amp; 不可重试异常(例如：磁盘满了、消息体积太大)</li>
</ul>
<h2 id="1-7-Kafka-之生产者重要参数详解"><a href="#1-7-Kafka-之生产者重要参数详解" class="headerlink" title="1.7 Kafka 之生产者重要参数详解"></a>1.7 Kafka 之生产者重要参数详解</h2><ul>
<li><p>acks: 指定发送消息后，Broker端至少有多少个副本接收到该消息；默认 acks = 1；（Broker端只要主分区写入成功，就可以给客户端去回送响应，<strong>如果leader宕机了，则会丢失数据</strong> ）</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/format,png.png" alt="这里写图片描述"></p>
</li>
<li><p>acks = 0：生产者发送消息之后不需要等待任何服务端的响应；（这种情况下数据传输效率最高，但是数据可靠性确是最低的。）</p>
</li>
<li><p>acks = -1 或者 acks=all：生产者在发送消息之后，需要等待 ISR 中的所有副本都成功写入消息之后才能够收到来自服务端的成功响应。</p>
<p>你以为这样就能保证数据不丢失了吗？例如当ISR中的成员只有leader的时候，就相当于 acks=1 了。</p>
<p>那么该怎么样保证数据的可靠性能？还需要<code>min.insync.replicas</code>这个参数(可以在broker或者topic层面进行设置)的配合，这样才能发挥最大的功效。</p>
<ul>
<li><p>min.insync.replicas这个参数设定ISR中的最小副本数是多少，默认值为1，当且仅当request.required.acks参数设置为-1时，此参数才生效。</p>
<p>如果ISR中的副本数少于<code>min.insync.replicas</code>配置的数量时，客户端会返回异常：org.apache.kafka.common.errors.NotEnoughReplicasExceptoin: Messages are rejected since there are fewer in-sync replicas than required。</p>
</li>
<li><p>ISR中的flower全部完成数据同步后，leader此时挂掉，会重新选举leader，数据不会丢失。</p>
</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/format,png-20210322221324553.png" alt="这里写图片描述"></p>
<ul>
<li>数据发送到leader后 ，部分ISR的副本同步，leader此时挂掉。比如follower1和follower2都有可能变成新的leader, producer端会得到返回异常，producer端会重新发送数据，数据可能会重复。</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/format,png-20210322221608062.png" alt="这里写图片描述"></p>
</li>
<li><p>max.request.size:该参数用来限制生产者客户端能发送的消息的最大值，默认 1M（10485768）</p>
</li>
<li><p>retries 和 retry.backoff.msretries: 重试次数和重试间隔，默认100</p>
</li>
<li><p>compression.type: 这个参数用来指定消息的压缩方式，默认值为 none ， 可选配置：gzip，snappy 和 lz4</p>
</li>
<li><p>connections.max.idle.ms:这个参数用来指定在多久之后关闭限制的连接，默认值是54000ms，即9分钟</p>
</li>
<li><p>linger.ms：这个参数用来指定生产者发送 ProducerBatch 之前等待更多消息（ProducerRecord）加入ProducerBatch的时间，默认值为0</p>
</li>
<li><p>batch.size:累计多少条消息，则一次进行批量发送</p>
</li>
<li><p>buffer.memory:缓存提升性能参数，默认 32 M</p>
</li>
<li><p>receive.buffer.bytes: 这个参数用来设置Socket接受消息缓冲区（SO_RECBUF）的大小，默认值为32678（B），即32KB</p>
</li>
<li><p>send.buffer.bytes: 这个参数用来设置Socket发送消息缓存区（SO_SNDBUF）的大小，默认值为131072（B），即128KB。</p>
</li>
<li><p>request.timeout.ms: 这个参数用来配置Producer等待请求响应的最长时间，默认值为 3000 ms</p>
</li>
</ul>
<h2 id="1-8-Kafka-之拦截器"><a href="#1-8-Kafka-之拦截器" class="headerlink" title="1.8 Kafka 之拦截器"></a>1.8 Kafka 之拦截器</h2><p>拦截器（interceptor）：Kafka对应着有生产者和消费者两种拦截器</p>
<p>生产者实现接口：org.apache.kafka.clients.producer.ProducerInterceptor</p>
<p>消费者实现接口：org.apache.kafka.clients.consumer.ConsumerInterceptor</p>
<h2 id="1-9-Kafka-之序列化和反序列化"><a href="#1-9-Kafka-之序列化和反序列化" class="headerlink" title="1.9 Kafka 之序列化和反序列化"></a>1.9 Kafka 之序列化和反序列化</h2><ul>
<li><p>序列化反序列化：生产者需要用序列化器（Serializer）把对象转换成字节数组才能通过网络发送Kafka；而在对侧，消费者需要用反序列化器（Derializer）把从Kafka中收到的字节数组转换成相应的对象。</p>
</li>
<li><p>序列化接口：<code>org.apache.kafka.common.serialization.Serializer</code></p>
<p>除了用于String类型的序列化器之外还有：ByteArray、ByteBuffer、Bytes、Double、Integer、Long这几种类型，它们都实现了org.apache.kafka.common.serialization.Serializer接口，此接口有三种方法：</p>
<ul>
<li><p>public void configure(Map&lt;String, ?&gt; configs, boolean isKey)：用来配置当前类。</p>
</li>
<li><p>public byte[] serialize(String topic, T data)：用来执行序列化。</p>
</li>
<li><p>public void close()：用来关闭当前序列化器。一般情况下这个方法都是个空方法，如果实现了此方法，必须确保此方法的幂等性，因为这个方法很可能会被KafkaProducer调用多次。</p>
</li>
</ul>
<p>如何自定义序列化？</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 要序列化的类</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="meta">@Data</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">User</span></span>&#123;</span><br><span class="line">  <span class="keyword">private</span> String id;</span><br><span class="line">  <span class="keyword">private</span> String name;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 序列化实现类</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">UserSerializer</span> <span class="keyword">implements</span> <span class="title">Serializer</span>&lt;<span class="title">User</span>&gt; </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Map&lt;String, ?&gt; configs, <span class="keyword">boolean</span> isKey)</span> </span>&#123;&#125;</span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">byte</span>[] serialize(String topic, User data) &#123;</span><br><span class="line">        <span class="keyword">if</span> (data == <span class="keyword">null</span>) &#123;</span><br><span class="line">            <span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">byte</span>[] id, name;</span><br><span class="line">        <span class="keyword">try</span> &#123;</span><br><span class="line">            <span class="keyword">if</span> (data.getId() != <span class="keyword">null</span>) &#123;</span><br><span class="line">                id = data.getId().getBytes(<span class="string">&quot;UTF-8&quot;</span>);</span><br><span class="line">            &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">                name = <span class="keyword">new</span> <span class="keyword">byte</span>[<span class="number">0</span>];</span><br><span class="line">            &#125;</span><br><span class="line">            <span class="keyword">if</span> (data.getName() != <span class="keyword">null</span>) &#123;</span><br><span class="line">                name = data.getName().getBytes(<span class="string">&quot;UTF-8&quot;</span>);</span><br><span class="line">            &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">                name = <span class="keyword">new</span> <span class="keyword">byte</span>[<span class="number">0</span>];</span><br><span class="line">            &#125;</span><br><span class="line">            ByteBuffer buffer = ByteBuffer.allocate(<span class="number">4</span>+<span class="number">4</span>+id.length + name.length);</span><br><span class="line">            buffer.putInt(id.length);</span><br><span class="line">            buffer.put(id);</span><br><span class="line">            buffer.putInt(name.length);</span><br><span class="line">            buffer.put(name);</span><br><span class="line">            <span class="keyword">return</span> buffer.array();</span><br><span class="line">        &#125; <span class="keyword">catch</span> (UnsupportedEncodingException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> <span class="keyword">byte</span>[<span class="number">0</span>];</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;&#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 使用: 记得也要将相应的String类型改为User类型，如：</span></span><br><span class="line">properties.put(<span class="string">&quot;value.serializer&quot;</span>, <span class="string">&quot;com.examples.fengxuechao.UserSerializer&quot;</span>);</span><br><span class="line">Producer&lt;String,User&gt; producer = <span class="keyword">new</span> KafkaProducer&lt;String,User&gt;(properties);</span><br><span class="line">User user = <span class="keyword">new</span> User(<span class="string">&quot;1&quot;</span>, <span class="string">&quot;Hi&quot;</span>);</span><br><span class="line">ProducerRecord&lt;String, User&gt; producerRecord = <span class="keyword">new</span> ProducerRecord&lt;String, User&gt;(topic,user);</span><br><span class="line"></span><br></pre></td></tr></table></figure></li>
<li><p>反序列化接口：<code>org.apache.kafka.common.serialization.Derializer</code></p>
<p>同接口同样有 3 个方法：</p>
<ul>
<li>public void configure(Map&lt;String, ?&gt; configs, boolean isKey)：用来配置当前类。</li>
<li>public byte[] serialize(String topic, T data)：用来执行反序列化。如果data为null建议处理的时候直接返回null而不是抛出一个异常。</li>
<li>public void close()：用来关闭当前序列化器。</li>
</ul>
<p>如何反序列化？</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">UserDeserializer</span> <span class="keyword">implements</span> <span class="title">Deserializer</span>&lt;<span class="title">User</span>&gt; </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Map&lt;String, ?&gt; configs, <span class="keyword">boolean</span> isKey)</span> </span>&#123;&#125;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> User <span class="title">deserialize</span><span class="params">(String topic, <span class="keyword">byte</span>[] data)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">if</span> (data == <span class="keyword">null</span>) &#123;</span><br><span class="line">            <span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">if</span> (data.length &lt; <span class="number">8</span>) &#123;</span><br><span class="line">            <span class="keyword">throw</span> <span class="keyword">new</span> SerializationException(<span class="string">&quot;Size of data received by UserDeserializer is shorter than expected!&quot;</span>);</span><br><span class="line">        &#125;</span><br><span class="line">        ByteBuffer buffer = ByteBuffer.wrap(data);</span><br><span class="line">        <span class="keyword">int</span> idLen, nameLen;</span><br><span class="line">        String id, name;</span><br><span class="line">        idLen = buffer.getInt();</span><br><span class="line">        <span class="keyword">byte</span>[] idBytes = <span class="keyword">new</span> <span class="keyword">byte</span>[idLen];</span><br><span class="line">        buffer.get(idBytes);</span><br><span class="line">        nameLen = buffer.getInt();</span><br><span class="line">        <span class="keyword">byte</span>[] nameBytes = <span class="keyword">new</span> <span class="keyword">byte</span>[nameLen];</span><br><span class="line">        buffer.get(nameBytes);</span><br><span class="line">        <span class="keyword">try</span> &#123;</span><br><span class="line">            id = <span class="keyword">new</span> String(idBytes, <span class="string">&quot;UTF-8&quot;</span>);</span><br><span class="line">            name = <span class="keyword">new</span> String(nameBytes, <span class="string">&quot;UTF-8&quot;</span>);</span><br><span class="line">        &#125; <span class="keyword">catch</span> (UnsupportedEncodingException e) &#123;</span><br><span class="line">            <span class="keyword">throw</span> <span class="keyword">new</span> SerializationException(<span class="string">&quot;Error occur when deserializing!&quot;</span>);</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> User(name,address);</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></li>
</ul>
<p>其实序列化完全可以和Avro、ProtoBuf等联合使用，而且更加的方便快捷。不过，如无必要，用默认的String序列化就可以了（使用自定义的序列化就不容易变了，如User类要添加一个属性）。</p>
<h2 id="1-10-Kafka-之分区器"><a href="#1-10-Kafka-之分区器" class="headerlink" title="1.10 Kafka 之分区器"></a>1.10 Kafka 之分区器</h2><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210325223028295.png" alt="image-20210325223028295"></p>
<p>上图是生产者发送消息后会经历一系列的过程：</p>
<ol>
<li>生产者发送消息</li>
<li>拦截器</li>
<li>序列化</li>
<li>分区：如果消息中没有指定分区，就会使用分区器</li>
<li>到达Broker</li>
</ol>
<p>生产者消息</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">ProducerRecord</span>&lt;<span class="title">K</span>, <span class="title">V</span>&gt; </span>&#123;</span><br><span class="line">    <span class="comment">// 所要发送的topic</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> String topic;       </span><br><span class="line">    <span class="comment">// 指定的partition序号</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Integer partition;  </span><br><span class="line">    <span class="comment">// 一组键值对，与RabbitMQ中的headers类似，kafka0.11.x版本才引入的一个属性</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Headers headers;    </span><br><span class="line">    <span class="comment">// 消息的key</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> K key;</span><br><span class="line">    <span class="comment">// 消息的value,即消息体</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> V value;</span><br><span class="line">    <span class="comment">// 消息的时间戳</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Long timestamp;</span><br><span class="line">    <span class="comment">// ...</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p> <code>org.apache.kafka:kafka-clients:2.0.1</code>中的 <code>KafkaProducer</code> 的<code>partition</code>源码如下：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * computes partition for given record.</span></span><br><span class="line"><span class="comment">     * if the record has partition returns the value otherwise</span></span><br><span class="line"><span class="comment">     * calls configured partitioner class to compute the partition.</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">int</span> <span class="title">partition</span><span class="params">(ProducerRecord&lt;K, V&gt; record, <span class="keyword">byte</span>[] serializedKey, <span class="keyword">byte</span>[] serializedValue, Cluster cluster)</span> </span>&#123;</span><br><span class="line">        Integer partition = record.partition();</span><br><span class="line">        <span class="keyword">return</span> partition != <span class="keyword">null</span> ?</span><br><span class="line">                partition :</span><br><span class="line">                partitioner.partition(</span><br><span class="line">                        record.topic(), record.key(), serializedKey, record.value(), serializedValue, cluster);</span><br><span class="line">    &#125;</span><br></pre></td></tr></table></figure>

<p>可以看出的确是先判断有无指明ProducerRecord的partition字段，如果没有指明，则再进一步计算分区。上面这段代码中的partitioner在默认情况下是指Kafka默认实现的<code>org.apache.kafka.clients.producer.DefaultPartitioner</code>，其源码如下：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * The default partitioning strategy:</span></span><br><span class="line"><span class="comment"> * &lt;ul&gt;</span></span><br><span class="line"><span class="comment"> * &lt;li&gt;If a partition is specified in the record, use it</span></span><br><span class="line"><span class="comment"> * &lt;li&gt;If no partition is specified but a key is present choose a partition based on a hash of the key</span></span><br><span class="line"><span class="comment"> * &lt;li&gt;If no partition or key is present choose a partition in a round-robin fashion</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">DefaultPartitioner</span> <span class="keyword">implements</span> <span class="title">Partitioner</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> ConcurrentMap&lt;String, AtomicInteger&gt; topicCounterMap = <span class="keyword">new</span> ConcurrentHashMap&lt;&gt;();</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Map&lt;String, ?&gt; configs)</span> </span>&#123;&#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * Compute the partition for the given record.</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> topic The topic name</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> key The key to partition on (or null if no key)</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> keyBytes serialized key to partition on (or null if no key)</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> value The value to partition on or null</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> valueBytes serialized value to partition on or null</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> cluster The current cluster metadata</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">int</span> <span class="title">partition</span><span class="params">(String topic, Object key, <span class="keyword">byte</span>[] keyBytes, Object value, <span class="keyword">byte</span>[] valueBytes, Cluster cluster)</span> </span>&#123;</span><br><span class="line">        List&lt;PartitionInfo&gt; partitions = cluster.partitionsForTopic(topic);</span><br><span class="line">        <span class="keyword">int</span> numPartitions = partitions.size();</span><br><span class="line">        <span class="keyword">if</span> (keyBytes == <span class="keyword">null</span>) &#123;</span><br><span class="line">            <span class="keyword">int</span> nextValue = nextValue(topic);</span><br><span class="line">            List&lt;PartitionInfo&gt; availablePartitions = cluster.availablePartitionsForTopic(topic);</span><br><span class="line">            <span class="keyword">if</span> (availablePartitions.size() &gt; <span class="number">0</span>) &#123;</span><br><span class="line">                <span class="keyword">int</span> part = Utils.toPositive(nextValue) % availablePartitions.size();</span><br><span class="line">                <span class="keyword">return</span> availablePartitions.get(part).partition();</span><br><span class="line">            &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">                <span class="comment">// no partitions are available, give a non-available partition</span></span><br><span class="line">                <span class="keyword">return</span> Utils.toPositive(nextValue) % numPartitions;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">            <span class="comment">// hash the keyBytes to choose a partition</span></span><br><span class="line">            <span class="keyword">return</span> Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">int</span> <span class="title">nextValue</span><span class="params">(String topic)</span> </span>&#123;</span><br><span class="line">        AtomicInteger counter = topicCounterMap.get(topic);</span><br><span class="line">        <span class="keyword">if</span> (<span class="keyword">null</span> == counter) &#123;</span><br><span class="line">            counter = <span class="keyword">new</span> AtomicInteger(ThreadLocalRandom.current().nextInt());</span><br><span class="line">            AtomicInteger currentCounter = topicCounterMap.putIfAbsent(topic, counter);</span><br><span class="line">            <span class="keyword">if</span> (currentCounter != <span class="keyword">null</span>) &#123;</span><br><span class="line">                counter = currentCounter;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> counter.getAndIncrement();</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;&#125;</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>由上源码可以看出partition的计算方式：</p>
<ol>
<li>如果key为null，则按照一种轮询的方式来计算分区分配</li>
<li>如果key不为null则使用称之为<code>murmur</code>的Hash算法（非加密型Hash函数，具备高运算性能及低碰撞率）来计算分区分配。</li>
</ol>
<p>当然我们也可自定义自己的分区器，如：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">UserPartitioner</span> <span class="keyword">implements</span> <span class="title">Partitioner</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> AtomicInteger atomicInteger = <span class="keyword">new</span> AtomicInteger(<span class="number">0</span>);</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Map&lt;String, ?&gt; configs)</span> </span>&#123;&#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">int</span> <span class="title">partition</span><span class="params">(String topic, Object key, <span class="keyword">byte</span>[] keyBytes, Object value, <span class="keyword">byte</span>[] valueBytes, Cluster cluster)</span> </span>&#123;</span><br><span class="line">        List&lt;PartitionInfo&gt; partitions = cluster.partitionsForTopic(topic);</span><br><span class="line">        <span class="keyword">int</span> numPartitions = partitions.size();</span><br><span class="line">        <span class="keyword">if</span> (<span class="keyword">null</span> == keyBytes || keyBytes.length&lt;<span class="number">1</span>) &#123;</span><br><span class="line">            <span class="keyword">return</span> atomicInteger.getAndIncrement() % numPartitions;</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">//借用String的hashCode的计算方式</span></span><br><span class="line">        <span class="keyword">int</span> hash = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">byte</span> b : keyBytes) &#123;</span><br><span class="line">            hash = <span class="number">31</span> * hash + b;</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> hash % numPartitions;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h2 id="1-11-Kafka-之消费者与消费者组"><a href="#1-11-Kafka-之消费者与消费者组" class="headerlink" title="1.11 Kafka 之消费者与消费者组"></a>1.11 Kafka 之消费者与消费者组</h2><h3 id="1-11-1-概念"><a href="#1-11-1-概念" class="headerlink" title="1.11.1 概念"></a>1.11.1 概念</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210325232006969.png" alt="image-20210325232006969"></p>
<p>说明：</p>
<ul>
<li><p>一个Topic可以有多个分区</p>
</li>
<li><p>一个主题可以有多个消费者组</p>
</li>
<li><p>一个消费者组可以有多个消费者，一个消费者只能属于一个消费者组</p>
</li>
<li><p>每一个分区可以被多个消费者组消费，每一个分区只能被一个消费者组中的一个消费者所消费，详见下图</p>
<p>  <img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210329224142774.png" alt="image-20210329224142774"></p>
<p>  <img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210329224552763.png" alt="image-20210329224552763"></p>
<p>  一个消费者组内的消费者数量多于分区时，多出来的消费者不做任何事。</p>
</li>
</ul>
<h3 id="1-11-2-消息中间件模型"><a href="#1-11-2-消息中间件模型" class="headerlink" title="1.11.2 消息中间件模型"></a>1.11.2 消息中间件模型</h3><ul>
<li><p>点对点（P2P，Point-to-Point）模式</p>
<p>点对点模式是基于队列的，消息生产者发送消息到队列，消息消费者从队列接受消息。</p>
</li>
<li><p>发布/订阅（Pub/Sub）模式</p>
<p>发布/订阅模式定义了如何向一个内容节点发布和订阅消息，这个内容节点成为主题（Topic），主题可以认为是消息传递的中介，消息发布者将消息发布到某个主题，而消息订阅者从主题中订阅消息</p>
</li>
<li><p>Kafka同时支持两种消息投递模式，而这正是得益于消费者与消费者组模型的契合</p>
<ul>
<li>所有的消费者都隶属于同一个消费组，相当于点对点模型</li>
<li>所有的消费者都隶属于不同的消费者组，相当于发布/订阅模型</li>
</ul>
</li>
</ul>
<h3 id="1-11-3-Kafka-消费者必要参数方法"><a href="#1-11-3-Kafka-消费者必要参数方法" class="headerlink" title="1.11.3 Kafka 消费者必要参数方法"></a>1.11.3 Kafka 消费者必要参数方法</h3><ul>
<li>bootstrap.servers: 用来指定连接Kafka集群所需的broker地址清单</li>
<li>key.deserializer 和 value.deserializer: 反序列化参数</li>
<li>group.id: 消费者所属消费组</li>
<li>subscribe：消费主题订阅，支持集合/标准正则表达式</li>
<li>assign：只订阅主题的某个分区</li>
</ul>
<h3 id="1-11-4-kafka-消费者提交位移"><a href="#1-11-4-kafka-消费者提交位移" class="headerlink" title="1.11.4 kafka 消费者提交位移"></a>1.11.4 kafka 消费者提交位移</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210331223055144.png" alt="image-20210331223055144"></p>
<p>在实际的工作中一般采用手动提交位移的方式，这样会有比较好的容错性，我们会知道这条消息到底有没有消费成功，如果处理失败，那我们可以再次提交等兜底的策略。</p>
<p><strong>Kafka 自动提交参数</strong></p>
<pre><code>- 自动提交：enable.auto.commit, 默认 true
- 提交周期间隔：auto.commit.interval.ms，默认值为 5 秒
</code></pre>
<p><strong>手工提交参数</strong></p>
<ul>
<li>enable.auto.commit，配置为 false</li>
<li>提交方式：commitSync &amp;commitAsync</li>
<li>同步提交：整体提交 &amp; 分区提交</li>
</ul>
<h3 id="1-11-5-消费者subscribe-与-assign-详解"><a href="#1-11-5-消费者subscribe-与-assign-详解" class="headerlink" title="1.11.5 消费者subscribe 与 assign 详解"></a>1.11.5 消费者subscribe 与 assign 详解</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210331233706053.png" alt="image-20210331233706053"></p>
<p>从上图中可以看到 subscribe 方法有 4 个重载的方法，对于 KafkaConsumer  消息的订阅，可以有多个主题，也可以支持正则表达式匹配。</p>
<p>假如我们只想要订阅一个 partition 呢？<br>使用 <code>assign</code> 方法</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 源码</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">PartitionInfo</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> String topic; <span class="comment">// 主题</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> <span class="keyword">int</span> partition; 分区</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Node leader; <span class="comment">// 主节点</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Node[] replicas; <span class="comment">// Kafka 节点</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Node[] inSyncReplicas; <span class="comment">// ISR Kafka 节点</span></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> Node[] offlineReplicas; <span class="comment">// OSR Kafka节点</span></span><br><span class="line">    </span><br><span class="line">    <span class="comment">// 省略方法</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 源码</span></span><br><span class="line"><span class="keyword">public</span> <span class="keyword">final</span> <span class="class"><span class="keyword">class</span> <span class="title">TopicPartition</span> <span class="keyword">implements</span> <span class="title">Serializable</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">int</span> hash = <span class="number">0</span>;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> <span class="keyword">int</span> partition;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> String topic;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"># 拉取某个主题下的所有分区</span><br><span class="line">List&lt;PartitionInfo&gt; tpInfoList = consumer.partitionsFor(<span class="string">&quot;topic&quot;</span>)</span><br><span class="line"># 订阅主题为 topic 的 第 0 个分区，0 是从 PartitionInfo 中取来的</span><br><span class="line">consumer.assign(Arrays.asList(<span class="keyword">new</span> TopicPartition(<span class="string">&quot;topic&quot;</span>, <span class="number">0</span>)))</span><br></pre></td></tr></table></figure>

<h3 id="1-11-6-Kafka消费者之多线程"><a href="#1-11-6-Kafka消费者之多线程" class="headerlink" title="1.11.6 Kafka消费者之多线程"></a>1.11.6 Kafka消费者之多线程</h3><ul>
<li>KafkaProducer 是线程安全的，但是KafkaConsumer却是线程非安全的</li>
<li>KafkaConsumer中定义了一个 <code>acquire</code>方法用来检测是否只有一个线程在操作，如果有其它线程操作则会抛出 ConcurrentModifactionException</li>
<li>KafkaConsumer在执行所有动作时都会先执行 <code>acquire</code> 方法检测是否线程安全</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406221942118.png" alt="image-20210406221942118"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406224758405.png" alt="image-20210406224758405"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210406230608886.png" alt="image-20210406230608886"></p>
<h3 id="1-11-7-Kafka-消费者重要参数"><a href="#1-11-7-Kafka-消费者重要参数" class="headerlink" title="1.11.7 Kafka 消费者重要参数"></a>1.11.7 Kafka 消费者重要参数</h3><p>性能调优参考</p>
<ul>
<li>fetch.min.bytes: 一次拉取最小数据量，默认1B</li>
<li>fetch.max.bytes: 一次拉取最大数据量，默认50M</li>
<li>max.partition.fetch.bytes: 一次fetch请求，从一个partition中取得的records最大大小，默认1M</li>
<li>fetch.max.wait.ms: Fetch 请求发给broker后，在broker中可能会被阻塞的时长，默认500</li>
<li>fetch.poll.records: Consumer 每次调用 poll() 时取到的records的最大数，默认 500 条</li>
</ul>
<h2 id="1-12-Kafka-高级应用整合-Spring-Boot"><a href="#1-12-Kafka-高级应用整合-Spring-Boot" class="headerlink" title="1.12 Kafka 高级应用整合 Spring Boot"></a>1.12 Kafka 高级应用整合 Spring Boot</h2><ol>
<li>Maven配置</li>
<li>application.properties</li>
<li>创建KafkaTemplate对象</li>
<li>@KafkaListener 监听消息</li>
</ol>
<h3 id="1-12-1-核心依赖"><a href="#1-12-1-核心依赖" class="headerlink" title="1.12.1 核心依赖"></a>1.12.1 核心依赖</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">parent</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.boot<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-boot-starter-parent<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">version</span>&gt;</span>2.1.5.RELEASE<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">relativePath</span>/&gt;</span> <span class="comment">&lt;!-- lookup parent from repository --&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">parent</span>&gt;</span>  </span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">dependencies</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">		<span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.boot<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">		<span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-boot-starter<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">		<span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.boot<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">		<span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-boot-starter-web<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;/<span class="name">dependency</span>&gt;</span> 				</span><br><span class="line">	<span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">	    <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.kafka<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">	    <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-kafka<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependencies</span>&gt;</span>	</span><br></pre></td></tr></table></figure>

<h3 id="1-12-2-生产者-application-properties-配置"><a href="#1-12-2-生产者-application-properties-配置" class="headerlink" title="1.12.2 生产者 application.properties 配置"></a>1.12.2 生产者 application.properties 配置</h3><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">## Spring 整合 kafka</span></span><br><span class="line"><span class="string">spring.kafka.bootstrap-servers=localhost:9092</span></span><br><span class="line"><span class="comment">## kafka producer 发送消息失败时的一个重试的次数</span></span><br><span class="line"><span class="string">spring.kafka.producer.retries=0</span></span><br><span class="line"><span class="comment">## 批量发送数据的配置 </span></span><br><span class="line"><span class="string">spring.kafka.producer.batch-size=16384</span></span><br><span class="line"><span class="comment">## 设置kafka 生产者内存缓存区的大小（32M）</span></span><br><span class="line"><span class="string">spring.kafka.producer.buffer-memory=33554432</span></span><br><span class="line"><span class="comment">## kafka消息的序列化配置</span></span><br><span class="line"><span class="string">spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer</span></span><br><span class="line"><span class="string">spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。</span></span><br><span class="line"><span class="comment"># acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。</span></span><br><span class="line"><span class="comment"># acks=-1: 表示分区leader必须等待消息被成功写入到所有的ISR副本(同步副本)中才认为producer请求成功。这种方案提供最高的消息持久性保证，但是理论上吞吐率也是最差的。</span></span><br><span class="line"></span><br><span class="line"><span class="comment">## 	这个是kafka生产端最重要的选项</span></span><br><span class="line"><span class="string">spring.kafka.producer.acks=1</span></span><br></pre></td></tr></table></figure>

<h3 id="1-12-3-KafkaProducerService"><a href="#1-12-3-KafkaProducerService" class="headerlink" title="1.12.3 KafkaProducerService"></a>1.12.3 KafkaProducerService</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Slf4j</span></span><br><span class="line"><span class="meta">@Component</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">KafkaProducerService</span> </span>&#123;</span><br><span class="line">	</span><br><span class="line">	<span class="meta">@Autowired</span></span><br><span class="line">	<span class="keyword">private</span> KafkaTemplate&lt;String, Object&gt; kafkaTemplate;</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">sendMessage</span><span class="params">(String topic, Object object)</span> </span>&#123;</span><br><span class="line">		</span><br><span class="line">		ListenableFuture&lt;SendResult&lt;String, Object&gt;&gt; future = kafkaTemplate.send(topic, object);</span><br><span class="line">		</span><br><span class="line">		future.addCallback(<span class="keyword">new</span> ListenableFutureCallback&lt;SendResult&lt;String, Object&gt;&gt;() &#123;</span><br><span class="line">			<span class="meta">@Override</span></span><br><span class="line">			<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">onSuccess</span><span class="params">(SendResult&lt;String, Object&gt; result)</span> </span>&#123;</span><br><span class="line">				log.info(<span class="string">&quot;发送消息成功: &quot;</span> + result.toString());</span><br><span class="line">			&#125;</span><br><span class="line"></span><br><span class="line">			<span class="meta">@Override</span></span><br><span class="line">			<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">onFailure</span><span class="params">(Throwable throwable)</span> </span>&#123;</span><br><span class="line">				log.error(<span class="string">&quot;发送消息失败: &quot;</span> + throwable.getMessage());</span><br><span class="line">			&#125;</span><br><span class="line">		&#125;);</span><br><span class="line">		</span><br><span class="line">	&#125;</span><br><span class="line">	</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h3 id="1-12-4-消费者-application-properties-配置"><a href="#1-12-4-消费者-application-properties-配置" class="headerlink" title="1.12.4 消费者 application.properties 配置"></a>1.12.4 消费者 application.properties 配置</h3><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">spring.kafka.bootstrap-servers=192.168.11.51:9092</span></span><br><span class="line"></span><br><span class="line"><span class="comment">## consumer 消息的签收机制：手工签收</span></span><br><span class="line"><span class="string">spring.kafka.consumer.enable-auto-commit=false</span></span><br><span class="line"><span class="string">spring.kafka.listener.ack-mode=manual</span></span><br><span class="line"><span class="comment"># 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理：</span></span><br><span class="line"><span class="comment"># latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录）</span></span><br><span class="line"><span class="comment"># earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录</span></span><br><span class="line"><span class="string">spring.kafka.consumer.auto-offset-reset=earliest</span></span><br><span class="line"><span class="comment">## 序列化配置</span></span><br><span class="line"><span class="string">spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer</span></span><br><span class="line"><span class="string">spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer</span></span><br><span class="line"></span><br><span class="line"><span class="string">spring.kafka.listener.concurrency=5</span></span><br></pre></td></tr></table></figure>

<h3 id="1-12-5-KafkaConsumerService"><a href="#1-12-5-KafkaConsumerService" class="headerlink" title="1.12.5 KafkaConsumerService"></a>1.12.5 KafkaConsumerService</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">@Slf4j</span><br><span class="line">@Component</span><br><span class="line">public class KafkaConsumerService &#123;</span><br><span class="line"></span><br><span class="line">   @KafkaListener(groupId &#x3D; &quot;group02&quot;, topics &#x3D; &quot;topic02&quot;)</span><br><span class="line">   public void onMessage(ConsumerRecord&lt;String, Object&gt; record, Acknowledgment acknowledgment, Consumer&lt;?, ?&gt; consumer) &#123;</span><br><span class="line">      log.info(&quot;消费端接收消息: &#123;&#125;&quot;, record.value());</span><br><span class="line">      &#x2F;&#x2F; 收工签收机制</span><br><span class="line">      acknowledgment.acknowledge();</span><br><span class="line">   &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h3 id="1-12-6-单元测试"><a href="#1-12-6-单元测试" class="headerlink" title="1.12.6 单元测试"></a>1.12.6 单元测试</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@RunWith(SpringRunner.class)</span></span><br><span class="line"><span class="meta">@SpringBootTest</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">ApplicationTests</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">	<span class="meta">@Autowired</span></span><br><span class="line">	<span class="keyword">private</span> KafkaProducerService kafkaProducerService;</span><br><span class="line">	</span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">send</span><span class="params">()</span> <span class="keyword">throws</span> InterruptedException </span>&#123;</span><br><span class="line">		</span><br><span class="line">		String topic = <span class="string">&quot;topic02&quot;</span>;</span><br><span class="line">		<span class="keyword">for</span>(<span class="keyword">int</span> i=<span class="number">0</span>; i &lt; <span class="number">1000</span>; i ++) &#123;</span><br><span class="line">			kafkaProducerService.sendMessage(topic, <span class="string">&quot;hello kafka&quot;</span> + i);</span><br><span class="line">			Thread.sleep(<span class="number">5</span>);</span><br><span class="line">		&#125;</span><br><span class="line">		</span><br><span class="line">	&#125;	</span><br><span class="line">	</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<h1 id="2-Kafka-海量日志收集系统实战"><a href="#2-Kafka-海量日志收集系统实战" class="headerlink" title="2 Kafka 海量日志收集系统实战"></a>2 Kafka 海量日志收集系统实战</h1><h2 id="2-1-架构设计"><a href="#2-1-架构设计" class="headerlink" title="2.1 架构设计"></a>2.1 架构设计</h2><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408215126380.png" alt="image-20210408215126380"></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408224353954.png" alt="image-20210408224353954"></p>
<p>说明：</p>
<ul>
<li><p>为什么不用SpringBoot默认的logback</p>
<p>因为log4j2 性能好</p>
</li>
<li><p>app.log 存储全量的日志，一般限制在 info 级别</p>
</li>
<li><p>error.log 存储 warn 级别以上的日志</p>
<p>方便后面做数据告警、分析，不选择 app.log 是因其日志太多</p>
</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408225148721.png" alt="image-20210408225148721"></p>
<p>说明：</p>
<ul>
<li>xpack-watch, trigger shell：通过触发器插件做一个对错误日志的上报和告警的功能</li>
</ul>
<h2 id="2-2-日志输出"><a href="#2-2-日志输出" class="headerlink" title="2.2 日志输出"></a>2.2 日志输出</h2><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210408230441675.png" alt="image-20210408230441675"></p>
<p>Log4j2.xml：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">Configuration</span> <span class="attr">status</span>=<span class="string">&quot;INFO&quot;</span> <span class="attr">schema</span>=<span class="string">&quot;Log4J-V2.0.xsd&quot;</span> <span class="attr">monitorInterval</span>=<span class="string">&quot;600&quot;</span> &gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">Properties</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">Property</span> <span class="attr">name</span>=<span class="string">&quot;LOG_HOME&quot;</span>&gt;</span>logs<span class="tag">&lt;/<span class="name">Property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span> <span class="attr">name</span>=<span class="string">&quot;FILE_NAME&quot;</span>&gt;</span>collector<span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%d&#123;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss.SSSZZ&#125;] : 用的是UTC时间，原因是 ELK 是UTC时间，为了保持一致 --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%level&#123;length=5&#125;] : 日志级别 --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%thread-%tid] : 线程ID --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%logger] : 日志输出的具体信息 --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%X&#123;hostName&#125;] [%X&#123;ip&#125;] [%X&#123;applicationName&#125;] : X 代表 MDC 自定义的日志输出 --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%F,%L,%C,%M] :  --&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- [%m] : message，代表要打印的日志内容--&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- &#x27;%ex&#x27;%n : ex 代表异常， %n 代表换行 --&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span> <span class="attr">name</span>=<span class="string">&quot;patternLayout&quot;</span>&gt;</span>[%d&#123;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss.SSSZZ&#125;] [%level&#123;length=5&#125;] [%thread-%tid] [%logger] [%X&#123;hostName&#125;] [%X&#123;ip&#125;] [%X&#123;applicationName&#125;] [%F,%L,%C,%M] [%m] ## &#x27;%ex&#x27;%n<span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">Properties</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">Appenders</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">Console</span> <span class="attr">name</span>=<span class="string">&quot;CONSOLE&quot;</span> <span class="attr">target</span>=<span class="string">&quot;SYSTEM_OUT&quot;</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">PatternLayout</span> <span class="attr">pattern</span>=<span class="string">&quot;$&#123;patternLayout&#125;&quot;</span>/&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">Console</span>&gt;</span>  </span><br><span class="line">        <span class="tag">&lt;<span class="name">RollingRandomAccessFile</span> <span class="attr">name</span>=<span class="string">&quot;appAppender&quot;</span> <span class="attr">fileName</span>=<span class="string">&quot;$&#123;LOG_HOME&#125;/app-$&#123;FILE_NAME&#125;.log&quot;</span> <span class="attr">filePattern</span>=<span class="string">&quot;$&#123;LOG_HOME&#125;/app-$&#123;FILE_NAME&#125;-%d&#123;yyyy-MM-dd&#125;-%i.log&quot;</span> &gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">PatternLayout</span> <span class="attr">pattern</span>=<span class="string">&quot;$&#123;patternLayout&#125;&quot;</span> /&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">Policies</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">TimeBasedTriggeringPolicy</span> <span class="attr">interval</span>=<span class="string">&quot;1&quot;</span>/&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">SizeBasedTriggeringPolicy</span> <span class="attr">size</span>=<span class="string">&quot;500MB&quot;</span>/&gt;</span></span><br><span class="line">          <span class="tag">&lt;/<span class="name">Policies</span>&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">DefaultRolloverStrategy</span> <span class="attr">max</span>=<span class="string">&quot;20&quot;</span>/&gt;</span>         </span><br><span class="line">        <span class="tag">&lt;/<span class="name">RollingRandomAccessFile</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">RollingRandomAccessFile</span> <span class="attr">name</span>=<span class="string">&quot;errorAppender&quot;</span> <span class="attr">fileName</span>=<span class="string">&quot;$&#123;LOG_HOME&#125;/error-$&#123;FILE_NAME&#125;.log&quot;</span> <span class="attr">filePattern</span>=<span class="string">&quot;$&#123;LOG_HOME&#125;/error-$&#123;FILE_NAME&#125;-%d&#123;yyyy-MM-dd&#125;-%i.log&quot;</span> &gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">PatternLayout</span> <span class="attr">pattern</span>=<span class="string">&quot;$&#123;patternLayout&#125;&quot;</span> /&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">Filters</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">ThresholdFilter</span> <span class="attr">level</span>=<span class="string">&quot;warn&quot;</span> <span class="attr">onMatch</span>=<span class="string">&quot;ACCEPT&quot;</span> <span class="attr">onMismatch</span>=<span class="string">&quot;DENY&quot;</span>/&gt;</span></span><br><span class="line">          <span class="tag">&lt;/<span class="name">Filters</span>&gt;</span>              </span><br><span class="line">          <span class="tag">&lt;<span class="name">Policies</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">TimeBasedTriggeringPolicy</span> <span class="attr">interval</span>=<span class="string">&quot;1&quot;</span>/&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">SizeBasedTriggeringPolicy</span> <span class="attr">size</span>=<span class="string">&quot;500MB&quot;</span>/&gt;</span></span><br><span class="line">          <span class="tag">&lt;/<span class="name">Policies</span>&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">DefaultRolloverStrategy</span> <span class="attr">max</span>=<span class="string">&quot;20&quot;</span>/&gt;</span>         </span><br><span class="line">        <span class="tag">&lt;/<span class="name">RollingRandomAccessFile</span>&gt;</span>            </span><br><span class="line">    <span class="tag">&lt;/<span class="name">Appenders</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">Loggers</span>&gt;</span></span><br><span class="line">        <span class="comment">&lt;!-- 业务相关 异步logger --&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">AsyncLogger</span> <span class="attr">name</span>=<span class="string">&quot;com.fengxuechao.examples.collector.*&quot;</span> <span class="attr">level</span>=<span class="string">&quot;info&quot;</span> <span class="attr">includeLocation</span>=<span class="string">&quot;true&quot;</span>&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">AppenderRef</span> <span class="attr">ref</span>=<span class="string">&quot;appAppender&quot;</span>/&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">AsyncLogger</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">AsyncLogger</span> <span class="attr">name</span>=<span class="string">&quot;com.fengxuechao.examples.collector.*&quot;</span> <span class="attr">level</span>=<span class="string">&quot;info&quot;</span> <span class="attr">includeLocation</span>=<span class="string">&quot;true&quot;</span>&gt;</span></span><br><span class="line">          <span class="tag">&lt;<span class="name">AppenderRef</span> <span class="attr">ref</span>=<span class="string">&quot;errorAppender&quot;</span>/&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">AsyncLogger</span>&gt;</span>       </span><br><span class="line">        <span class="tag">&lt;<span class="name">Root</span> <span class="attr">level</span>=<span class="string">&quot;info&quot;</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">Appender-Ref</span> <span class="attr">ref</span>=<span class="string">&quot;CONSOLE&quot;</span>/&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">Appender-Ref</span> <span class="attr">ref</span>=<span class="string">&quot;appAppender&quot;</span>/&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">AppenderRef</span> <span class="attr">ref</span>=<span class="string">&quot;errorAppender&quot;</span>/&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">Root</span>&gt;</span>         </span><br><span class="line">    <span class="tag">&lt;/<span class="name">Loggers</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">Configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h2 id="2-3-日志收集"><a href="#2-3-日志收集" class="headerlink" title="2.3 日志收集"></a>2.3 日志收集</h2><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210419220943341.png" alt="image-20210419220943341"></p>
<p>filebeat.yml配置文件</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">filebeat.inputs:</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Each - is an input. Most options can be set at the input level, so</span></span><br><span class="line"><span class="comment"># you can use different inputs for various configurations.</span></span><br><span class="line"><span class="comment"># Below are the input specific configurations.</span></span><br><span class="line"></span><br><span class="line"><span class="bullet">-</span> <span class="attr">type:</span> <span class="string">log</span></span><br><span class="line"></span><br><span class="line">  <span class="comment"># Change to true to enable this input configuration.</span></span><br><span class="line">  <span class="attr">enabled:</span> <span class="literal">true</span></span><br><span class="line"></span><br><span class="line">  <span class="comment"># Paths that should be crawled and fetched. Glob based paths.</span></span><br><span class="line">  <span class="attr">paths:</span></span><br><span class="line">    <span class="comment"># - /var/log/*.log</span></span><br><span class="line">    <span class="comment">#- c:\programdata\elasticsearch\logs\*</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># app-服务名.log, 为什么写死, 防止发生轮转抓取历史数据</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">/usr/local/var/foodie/logs/app-collector.log</span></span><br><span class="line">  <span class="comment"># 定义写入ES时的 _type 值</span></span><br><span class="line">  <span class="attr">document_type:</span> <span class="string">&quot;app-log&quot;</span></span><br><span class="line">  <span class="comment"># 指定匹配的表达式（匹配以 [ 开头的字符串）</span></span><br><span class="line">  <span class="attr">multiline:</span></span><br><span class="line">    <span class="comment"># 指定匹配的表达式（匹配以 [ 开头的字符串）</span></span><br><span class="line">    <span class="attr">pattern:</span> <span class="string">&#x27;^\[&#x27;</span></span><br><span class="line">    <span class="comment"># 是否匹配到</span></span><br><span class="line">    <span class="attr">negate:</span> <span class="literal">true</span></span><br><span class="line">    <span class="comment"># 合并到上一行的末尾</span></span><br><span class="line">    <span class="attr">match:</span> <span class="string">after</span></span><br><span class="line">    <span class="comment"># 最大的行数</span></span><br><span class="line">    <span class="attr">max_lines:</span> <span class="number">2000</span></span><br><span class="line">    <span class="comment"># 如果在规定的时间没有新的日志事件就不等待后面的日志</span></span><br><span class="line">    <span class="attr">timeout:</span> <span class="string">2s</span></span><br><span class="line">  <span class="attr">fields:</span></span><br><span class="line">    <span class="attr">logbiz:</span> <span class="string">collector</span></span><br><span class="line">    <span class="attr">logtopic:</span> <span class="string">app-log-collector</span></span><br><span class="line">    <span class="attr">evn:</span> <span class="string">dev</span></span><br><span class="line"></span><br><span class="line"><span class="bullet">-</span> <span class="attr">type:</span> <span class="string">log</span></span><br><span class="line">  <span class="attr">enabled:</span> <span class="literal">true</span> </span><br><span class="line">  <span class="attr">paths:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">/usr/local/var/foodie/logs/error-collector.log</span></span><br><span class="line">  <span class="comment"># 定义写入ES时的 _type 值</span></span><br><span class="line">  <span class="attr">document_type:</span> <span class="string">&quot;error-log&quot;</span></span><br><span class="line">  <span class="comment"># 指定匹配的表达式（匹配以 [ 开头的字符串）</span></span><br><span class="line">  <span class="attr">multiline:</span></span><br><span class="line">    <span class="comment"># 指定匹配的表达式（匹配以 [ 开头的字符串）</span></span><br><span class="line">    <span class="attr">pattern:</span> <span class="string">&#x27;^\[&#x27;</span></span><br><span class="line">    <span class="comment"># 是否匹配到</span></span><br><span class="line">    <span class="attr">negate:</span> <span class="literal">true</span></span><br><span class="line">    <span class="comment"># 合并到上一行的末尾</span></span><br><span class="line">    <span class="attr">match:</span> <span class="string">after</span></span><br><span class="line">    <span class="comment"># 最大的行数</span></span><br><span class="line">    <span class="attr">max_lines:</span> <span class="number">2000</span></span><br><span class="line">    <span class="comment"># 如果在规定的时间没有新的日志事件就不等待后面的日志</span></span><br><span class="line">    <span class="attr">timeout:</span> <span class="string">2s</span></span><br><span class="line">  <span class="attr">fields:</span></span><br><span class="line">    <span class="attr">logbiz:</span> <span class="string">collector</span></span><br><span class="line">    <span class="attr">logtopic:</span> <span class="string">error-log-collector</span></span><br><span class="line">    <span class="attr">evn:</span> <span class="string">dev</span></span><br></pre></td></tr></table></figure>

<p>检查配置文件是否正确</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./filebeat -c filebeat.yml -configtest</span><br></pre></td></tr></table></figure>

<p>启动filebeat</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">/filebeat &amp;</span><br></pre></td></tr></table></figure>

<h2 id="2-4-日志过滤"><a href="#2-4-日志过滤" class="headerlink" title="2.4 日志过滤"></a>2.4 日志过滤</h2><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210420223154642.png" alt="image-20210420223154642"></p>
<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/huhangfei/p/7605511.html">Logstash使用介绍</a></p>
<h2 id="2-5-日志持久化、可视化"><a href="#2-5-日志持久化、可视化" class="headerlink" title="2.5 日志持久化、可视化"></a>2.5 日志持久化、可视化</h2><ol>
<li><p>ElasticSearch 索引创建周期、命名规范选择</p>
<p>logstash 配置文件</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">input</span> &#123;</span><br><span class="line">  <span class="string">kafka</span> &#123;</span><br><span class="line">    <span class="comment">## error-log-服务名称</span></span><br><span class="line">    <span class="string">topics_pattern</span> <span class="string">=&gt;</span> <span class="string">&quot;error-log-.*&quot;</span></span><br><span class="line">    <span class="string">bootstrap_servers</span> <span class="string">=&gt;</span> <span class="string">&quot;192.168.11.51:9092&quot;</span></span><br><span class="line">    <span class="string">codec</span> <span class="string">=&gt;</span> <span class="string">json</span></span><br><span class="line">    <span class="string">consumer_threads</span> <span class="string">=&gt;</span> <span class="number">4</span></span><br><span class="line">    <span class="string">decorate_events</span> <span class="string">=&gt;</span> <span class="literal">true</span></span><br><span class="line">    <span class="comment"># auto_offset_rest =&gt; &quot;latest&quot;</span></span><br><span class="line">    <span class="string">group_id</span> <span class="string">=&gt;</span> <span class="string">&quot;error-log-group&quot;</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="string">filter</span> &#123;</span><br><span class="line">  <span class="comment">## 时区转换, 每天创建一个索引</span></span><br><span class="line">  <span class="string">ruby</span> &#123;</span><br><span class="line">    <span class="string">code</span> <span class="string">=&gt;</span> <span class="string">&quot;event.set(&#x27;index_time&#x27;, event.timestamp.time.localtime.strftime(&#x27;%Y.%m.%d&#x27;))&quot;</span></span><br><span class="line">  &#125;</span><br><span class="line">  <span class="string">if</span> <span class="string">&quot;app-log&quot;</span> <span class="string">in</span> [<span class="string">fields</span>][<span class="string">logtopic</span>] &#123;</span><br><span class="line">    <span class="string">grok</span> &#123;</span><br><span class="line">      <span class="comment">## 表达式</span></span><br><span class="line">      <span class="string">match</span> <span class="string">=&gt;</span> [<span class="string">&quot;message&quot;</span>,<span class="string">&quot;\[<span class="template-variable">%&#123;NOTSPACE:currentDateTime&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:level&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:thread-id&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:class&#125;</span>\] \[<span class="template-variable">%&#123;DATA:hostName&#125;</span>\] \[<span class="template-variable">%&#123;DATA:ip&#125;</span>\] \[<span class="template-variable">%&#123;DATA:applicationName&#125;</span>\] \[<span class="template-variable">%&#123;DATA:location&#125;</span>\] \[<span class="template-variable">%&#123;DATA:messageInfo&#125;</span>\] ## (\&#x27;\&#x27;|<span class="template-variable">%&#123;QUOTEDSTRING:throwable&#125;</span>)&quot;</span>]</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">  <span class="string">if</span> <span class="string">&quot;error-log&quot;</span> <span class="string">in</span> [<span class="string">fields</span>][<span class="string">logtopic</span>] &#123;</span><br><span class="line">    <span class="string">grok</span> &#123;</span><br><span class="line">      <span class="comment">## 表达式</span></span><br><span class="line">      <span class="string">match</span> <span class="string">=&gt;</span> [<span class="string">&quot;message&quot;</span>,<span class="string">&quot;\[<span class="template-variable">%&#123;NOTSPACE:currentDateTime&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:level&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:thread-id&#125;</span>\] \[<span class="template-variable">%&#123;NOTSPACE:class&#125;</span>\] \[<span class="template-variable">%&#123;DATA:hostName&#125;</span>\] \[<span class="template-variable">%&#123;DATA:ip&#125;</span>\] \[<span class="template-variable">%&#123;DATA:applicationName&#125;</span>\] \[<span class="template-variable">%&#123;DATA:location&#125;</span>\] \[<span class="template-variable">%&#123;DATA:messageInfo&#125;</span>\] ## (\&#x27;\&#x27;|<span class="template-variable">%&#123;QUOTEDSTRING:throwable&#125;</span>)&quot;</span>]</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="string">output</span> &#123;</span><br><span class="line">  <span class="string">if</span> <span class="string">&quot;app-log&quot;</span> <span class="string">in</span> [<span class="string">fields</span>][<span class="string">logtopic</span>] &#123;</span><br><span class="line">    <span class="comment">## es 插件</span></span><br><span class="line">    <span class="string">elasticsearch</span> &#123;</span><br><span class="line">      <span class="string">hosts</span> <span class="string">=&gt;</span> [<span class="string">&quot;192.168.11.35:9200&quot;</span>]</span><br><span class="line">      <span class="string">user</span> <span class="string">=&gt;</span> <span class="string">&quot;elastic&quot;</span></span><br><span class="line">      <span class="string">password</span> <span class="string">=&gt;</span> <span class="string">&quot;123456&quot;</span></span><br><span class="line">      <span class="comment">## 索引名。+ 号开头的就会自动认为后面是时间格式</span></span><br><span class="line">      <span class="string">index</span> <span class="string">=&gt;</span> <span class="string">&quot;app-log-<span class="template-variable">%&#123;[fields][logbiz]&#125;</span>-<span class="template-variable">%&#123;index_time&#125;</span>&quot;</span></span><br><span class="line">      <span class="comment"># 是否嗅探集群ip：一般设置 true：http://192.168.11.35:9200/_nodes/http?pretty</span></span><br><span class="line">      <span class="comment"># 通过嗅探机制进行 es 集群负载均衡发日志消息</span></span><br><span class="line">      <span class="string">sniffing</span> <span class="string">=&gt;</span> <span class="literal">true</span></span><br><span class="line">      <span class="comment"># logstash 默认自带一个mapping模版.进行模版覆盖</span></span><br><span class="line">      <span class="string">template_overwrite</span> <span class="string">=&gt;</span> <span class="literal">true</span></span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">  <span class="string">if</span> <span class="string">&quot;error-log&quot;</span> <span class="string">in</span> [<span class="string">fields</span>][<span class="string">logtopic</span>] &#123;</span><br><span class="line">    <span class="comment">## es 插件</span></span><br><span class="line">    <span class="string">elasticsearch</span> &#123;</span><br><span class="line">      <span class="string">hosts</span> <span class="string">=&gt;</span> [<span class="string">&quot;192.168.11.35:9200&quot;</span>]</span><br><span class="line">      <span class="string">user</span> <span class="string">=&gt;</span> <span class="string">&quot;elastic&quot;</span></span><br><span class="line">      <span class="string">password</span> <span class="string">=&gt;</span> <span class="string">&quot;123456&quot;</span></span><br><span class="line">      <span class="comment">## 索引名。+ 号开头的就会自动认为后面是时间格式</span></span><br><span class="line">      <span class="string">index</span> <span class="string">=&gt;</span> <span class="string">&quot;app-log-<span class="template-variable">%&#123;[fields][logbiz]&#125;</span>-<span class="template-variable">%&#123;index_time&#125;</span>&quot;</span></span><br><span class="line">      <span class="comment"># 是否嗅探集群ip：一般设置 true：http://192.168.11.35:9200/_nodes/http?pretty</span></span><br><span class="line">      <span class="comment"># 通过嗅探机制进行 es 集群负载均衡发日志消息</span></span><br><span class="line">      <span class="string">sniffing</span> <span class="string">=&gt;</span> <span class="literal">true</span></span><br><span class="line">      <span class="comment"># logstash 默认自带一个mapping模版.进行模版覆盖</span></span><br><span class="line">      <span class="string">template_overwrite</span> <span class="string">=&gt;</span> <span class="literal">true</span></span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></li>
<li><p>Kibana 控制台应用、可视化日志</p>
</li>
<li><p>监控和告警</p>
<ol>
<li><p>Watcher 插件作用介绍基本使用</p>
<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/lhxsoft/p/13089381.html">ElasticSearch(ES)预警服务 Watcher安装以及探究</a></p>
<p>示例：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br></pre></td><td class="code"><pre><span class="line">## 创建一个watcher,比如定义一个trigger 每个10s钟看一下input里的数据</span><br><span class="line">## 创建一个watcher,比如定义一个trigger 每个5s钟看一下input里的数据</span><br><span class="line">PUT _xpack&#x2F;watcher&#x2F;watch&#x2F;error_log_collector_watcher</span><br><span class="line">&#123;</span><br><span class="line">  &quot;trigger&quot;: &#123;</span><br><span class="line">    &quot;schedule&quot;: &#123;</span><br><span class="line">      &quot;interval&quot;: &quot;5s&quot;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;,</span><br><span class="line">  &quot;input&quot;: &#123;</span><br><span class="line">    &quot;search&quot;: &#123;</span><br><span class="line">      &quot;request&quot;: &#123;</span><br><span class="line">        &quot;indices&quot;: [&quot;&lt;error_log_collector-&#123;now+8h&#x2F;d&#125;&gt;&quot;],</span><br><span class="line">        &quot;body&quot;: &#123;</span><br><span class="line">          &quot;size&quot;: 0,</span><br><span class="line">          &quot;query&quot;: &#123;</span><br><span class="line">            &quot;bool&quot;: &#123;</span><br><span class="line">              &quot;must&quot;: [</span><br><span class="line">                  &#123;</span><br><span class="line">                    &quot;term&quot;: &#123;&quot;level&quot;: &quot;ERROR&quot;&#125;</span><br><span class="line">                  &#125;</span><br><span class="line">              ],</span><br><span class="line">              &quot;filter&quot;: &#123;</span><br><span class="line">                &quot;range&quot;: &#123;</span><br><span class="line">                    &quot;currentDateTime&quot;: &#123;</span><br><span class="line">                    &quot;gt&quot;: &quot;now-30s&quot; , &quot;lt&quot;: &quot;now&quot;</span><br><span class="line">                  &#125;</span><br><span class="line">                &#125;</span><br><span class="line">              &#125; </span><br><span class="line">            &#125;</span><br><span class="line">          &#125;</span><br><span class="line">        &#125;</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;,</span><br><span class="line"></span><br><span class="line">  &quot;condition&quot;: &#123;</span><br><span class="line">    &quot;compare&quot;: &#123;</span><br><span class="line">      &quot;ctx.payload.hits.total&quot;: &#123;</span><br><span class="line">        &quot;gt&quot;: 0</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;,</span><br><span class="line"> </span><br><span class="line">  &quot;transform&quot;: &#123;</span><br><span class="line">    &quot;search&quot;: &#123;</span><br><span class="line">      &quot;request&quot;: &#123;</span><br><span class="line">        &quot;indices&quot;: [&quot;&lt;error-log-collector-&#123;now+8h&#x2F;d&#125;&gt;&quot;],</span><br><span class="line">        &quot;body&quot;: &#123;</span><br><span class="line">          &quot;size&quot;: 1,</span><br><span class="line">          &quot;query&quot;: &#123;</span><br><span class="line">            &quot;bool&quot;: &#123;</span><br><span class="line">              &quot;must&quot;: [</span><br><span class="line">                  &#123;</span><br><span class="line">                    &quot;term&quot;: &#123;&quot;level&quot;: &quot;ERROR&quot;&#125;</span><br><span class="line">                  &#125;</span><br><span class="line">              ],</span><br><span class="line">              &quot;filter&quot;: &#123;</span><br><span class="line">                &quot;range&quot;: &#123;</span><br><span class="line">                    &quot;currentDateTime&quot;: &#123;</span><br><span class="line">                    &quot;gt&quot;: &quot;now-30s&quot; , &quot;lt&quot;: &quot;now&quot;</span><br><span class="line">                  &#125;</span><br><span class="line">                &#125;</span><br><span class="line">              &#125; </span><br><span class="line">            &#125;</span><br><span class="line">          &#125;,</span><br><span class="line">          &quot;sort&quot;: [</span><br><span class="line">            &#123;</span><br><span class="line">                &quot;currentDateTime&quot;: &#123;</span><br><span class="line">                    &quot;order&quot;: &quot;desc&quot;</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">          ]</span><br><span class="line">        &#125;</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;,</span><br><span class="line">  &quot;actions&quot;: &#123;</span><br><span class="line">    &quot;test_error&quot;: &#123;</span><br><span class="line">      &quot;webhook&quot; : &#123;</span><br><span class="line">        &quot;method&quot; : &quot;POST&quot;,</span><br><span class="line">        &quot;url&quot; : &quot;http:&#x2F;&#x2F;192.168.11.31:8001&#x2F;accurateWatch&quot;,</span><br><span class="line">        &quot;body&quot; : &quot;&#123;\&quot;title\&quot;: \&quot;异常错误告警\&quot;, \&quot;applicationName\&quot;: \&quot;&#123;&#123;#ctx.payload.hits.hits&#125;&#125;&#123;&#123;_source.applicationName&#125;&#125;&#123;&#123;&#x2F;ctx.payload.hits.hits&#125;&#125;\&quot;, \&quot;level\&quot;:\&quot;告警级别P1\&quot;, \&quot;body\&quot;: \&quot;&#123;&#123;#ctx.payload.hits.hits&#125;&#125;&#123;&#123;_source.messageInfo&#125;&#125;&#123;&#123;&#x2F;ctx.payload.hits.hits&#125;&#125;\&quot;, \&quot;executionTime\&quot;: \&quot;&#123;&#123;#ctx.payload.hits.hits&#125;&#125;&#123;&#123;_source.currentDateTime&#125;&#125;&#123;&#123;&#x2F;ctx.payload.hits.hits&#125;&#125;\&quot;&#125;&quot;</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line"> &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"># 查看一个watcher</span><br><span class="line"># </span><br><span class="line">GET _xpack&#x2F;watcher&#x2F;watch&#x2F;error_log_collector_watcher</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">#删除一个watcher</span><br><span class="line">DELETE _xpack&#x2F;watcher&#x2F;watch&#x2F;error_log_collector_watcher</span><br><span class="line"></span><br><span class="line">#执行watcher</span><br><span class="line"># POST _xpack&#x2F;watcher&#x2F;watch&#x2F;error_log_collector_watcher&#x2F;_execute</span><br><span class="line"></span><br><span class="line">#查看执行结果</span><br><span class="line">GET &#x2F;.watcher-history*&#x2F;_search?pretty</span><br><span class="line">&#123;</span><br><span class="line">  &quot;sort&quot; : [</span><br><span class="line">    &#123; &quot;result.execution_time&quot; : &quot;desc&quot; &#125;</span><br><span class="line">  ],</span><br><span class="line">  &quot;query&quot;: &#123;</span><br><span class="line">    &quot;match&quot;: &#123;</span><br><span class="line">      &quot;watch_id&quot;: &quot;error_log_collector_watcher&quot;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">GET error-log-collector-2019.09.18&#x2F;_search?size&#x3D;10</span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">  &quot;query&quot;: &#123;</span><br><span class="line">    &quot;match&quot;: &#123;</span><br><span class="line">      &quot;level&quot;: &quot;ERROR&quot;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">  ,</span><br><span class="line">  &quot;sort&quot;: [</span><br><span class="line">    &#123;</span><br><span class="line">        &quot;currentDateTime&quot;: &#123;</span><br><span class="line">            &quot;order&quot;: &quot;desc&quot;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  ] </span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">GET error-log-collector-2019.09.18&#x2F;_search?size&#x3D;10</span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">  &quot;query&quot;: &#123;</span><br><span class="line">    &quot;match&quot;: &#123;</span><br><span class="line">      &quot;level&quot;: &quot;ERROR&quot;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">  ,</span><br><span class="line">  &quot;sort&quot;: [</span><br><span class="line">    &#123;</span><br><span class="line">        &quot;currentDateTime&quot;: &#123;</span><br><span class="line">            &quot;order&quot;: &quot;desc&quot;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  ] </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></li>
<li><p>Watcher API 详解</p>
</li>
<li><p>Watcher 实战应用告警</p>
</li>
</ol>
</li>
</ol>
<h1 id="3-Kafka-数据同步"><a href="#3-Kafka-数据同步" class="headerlink" title="3 Kafka 数据同步"></a>3 Kafka 数据同步</h1><h2 id="3-1-什么是数据同步？"><a href="#3-1-什么是数据同步？" class="headerlink" title="3.1 什么是数据同步？"></a>3.1 什么是数据同步？</h2><blockquote>
<p>转载自<a target="_blank" rel="noopener" href="https://www.cnblogs.com/binghe001/p/13445117.html">https://www.cnblogs.com/binghe001/p/13445117.html</a></p>
</blockquote>
<p>在当今互联网行业，尤其是现在分布式、微服务开发环境下，为了提高搜索效率，以及搜索的精准度，会大量使用Redis、Memcached等NoSQL数据库，也会使用大量的Solr、Elasticsearch等全文检索服务。那么，这个时候，就会有一个问题需要我们来思考和解决：那就是数据同步的问题！如何将实时变化的数据库中的数据同步到Redis/Memcached或者Solr/Elasticsearch中呢？</p>
<p>例如，我们在分布式环境下向数据库中不断的写入数据，而我们读数据可能需要从Redis、Memcached或者Elasticsearch、Solr等服务中读取。那么，数据库与各个服务中数据的实时同步问题，成为了我们亟待解决的问题。</p>
<p>试想，由于业务需要，我们引入了Redis、Memcached或者Elasticsearch、Solr等服务。使得我们的应用程序可能会从不同的服务中读取数据，如下图所示。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110610852.png" alt="20200806110610852"></p>
<p>本质上讲，无论我们引入了何种服务或者中间件，数据最终都是从我们的MySQL数据库中读取出来的。那么，问题来了，如何将MySQL中的数据实时同步到其他的服务或者中间件呢？</p>
<h2 id="3-2-如何去选择数据同步技术？"><a href="#3-2-如何去选择数据同步技术？" class="headerlink" title="3.2 如何去选择数据同步技术？"></a>3.2 如何去选择数据同步技术？</h2><h3 id="3-2-1-在业务代码中同步"><a href="#3-2-1-在业务代码中同步" class="headerlink" title="3.2.1 在业务代码中同步"></a>3.2.1 在业务代码中同步</h3><p>在增加、修改、删除之后，执行操作Solr索引库的逻辑代码。例如下面的代码片段。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">public</span> ResponseResult <span class="title">updateStatus</span><span class="params">(Long[] ids, String status)</span></span>&#123;</span><br><span class="line">    <span class="keyword">try</span>&#123;</span><br><span class="line">        goodsService.updateStatus(ids, status);</span><br><span class="line">        <span class="keyword">if</span>(<span class="string">&quot;status_success&quot;</span>.equals(status))&#123;</span><br><span class="line">            List&lt;TbItem&gt; itemList = goodsService.getItemList(ids, status);</span><br><span class="line">            itemSearchService.importList(itemList);</span><br><span class="line">            <span class="keyword">return</span> <span class="keyword">new</span> ResponseResult(<span class="keyword">true</span>, <span class="string">&quot;修改状态成功&quot;</span>)</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;<span class="keyword">catch</span>(Exception e)&#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> ResponseResult(<span class="keyword">false</span>, <span class="string">&quot;修改状态失败&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>优点：</strong></p>
<p>操作简便。</p>
<p><strong>缺点：</strong></p>
<p>业务耦合度高。</p>
<p>执行效率变低。</p>
<h3 id="3-2-2-定时任务同步"><a href="#3-2-2-定时任务同步" class="headerlink" title="3.2.2 定时任务同步"></a>3.2.2 定时任务同步</h3><p>在数据库中执行完增加、修改、删除操作后，通过定时任务定时的将数据库的数据同步到Solr索引库中。</p>
<p>定时任务技术有：SpringTask，Quartz。</p>
<p><strong>这里执行定时任务时，需要注意的一个技巧是：第一次执行定时任务时，从MySQL数据库中以时间字段进行倒序排列查询相应的数据，并记录当前查询数据的时间字段的最大值，以后每次执行定时任务查询数据的时候，只要按时间字段倒序查询数据表中的时间字段大于上次记录的时间值的数据，并且记录本次任务查询出的时间字段的最大值即可，从而不需要再次查询数据表中的所有数据。</strong></p>
<p><strong>注意：这里所说的时间字段指的是标识数据更新的时间字段，也就是说，使用定时任务同步数据时，为了避免每次执行任务都会进行全表扫描，最好是在数据表中增加一个更新记录的时间字段。</strong></p>
<p><strong>优点：</strong></p>
<p>同步Solr索引库的操作与业务代码完全解耦。</p>
<p><strong>缺点：</strong></p>
<p>数据的实时性并不高。</p>
<h3 id="3-2-3-通过MQ实现同步"><a href="#3-2-3-通过MQ实现同步" class="headerlink" title="3.2.3 通过MQ实现同步"></a>3.2.3 通过MQ实现同步</h3><p>在数据库中执行完增加、修改、删除操作后，向MQ中发送一条消息，此时，同步程序作为MQ中的消费者，从消息队列中获取消息，然后执行同步Solr索引库的逻辑。</p>
<p>我们可以使用下图来简单的标识通过MQ实现数据同步的过程。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110631185.png" alt="img"></p>
<p>我们可以使用如下代码实现这个过程。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">public</span> ResponseResult <span class="title">updateStatus</span><span class="params">(Long[] ids, String status)</span></span>&#123;</span><br><span class="line">    <span class="keyword">try</span>&#123;</span><br><span class="line">        goodsService.updateStatus(ids, status);</span><br><span class="line">        <span class="keyword">if</span>(<span class="string">&quot;status_success&quot;</span>.equals(status))&#123;</span><br><span class="line">            List&lt;TbItem&gt; itemList = goodsService.getItemList(ids, status);</span><br><span class="line">            <span class="keyword">final</span> String jsonString = JSON.toJSONString(itemList);</span><br><span class="line">            jmsTemplate.send(queueSolr, <span class="keyword">new</span> MessageCreator()&#123;</span><br><span class="line">                <span class="meta">@Override</span></span><br><span class="line">                <span class="function"><span class="keyword">public</span> Message <span class="title">createMessage</span><span class="params">(Session session)</span> <span class="keyword">throws</span> JMSException</span>&#123;</span><br><span class="line">                    <span class="keyword">return</span> session.createTextMessage(jsonString);</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;);</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> ResponseResult(<span class="keyword">true</span>, <span class="string">&quot;修改状态成功&quot;</span>);</span><br><span class="line">    &#125;<span class="keyword">catch</span>(Exception e)&#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> ResponseResult(<span class="keyword">false</span>, <span class="string">&quot;修改状态失败&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>优点：</strong></p>
<p>业务代码解耦，并且能够做到准实时。</p>
<p><strong>缺点：</strong></p>
<p>需要在业务代码中加入发送消息到MQ的代码，数据调用接口耦合。</p>
<h3 id="3-2-4-通过Canal实现实时同步"><a href="#3-2-4-通过Canal实现实时同步" class="headerlink" title="3.2.4 通过Canal实现实时同步"></a>3.2.4 通过Canal实现实时同步</h3><p>Canal是阿里巴巴开源的一款数据库日志增量解析组件，通过Canal来解析数据库的日志信息，来检测数据库中表结构和数据的变化，从而更新Solr索引库。</p>
<p>使用Canal可以做到业务代码完全解耦，API完全解耦，可以做到准实时。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210426224407408.png" alt="image-20210426224407408"></p>
<h2 id="3-3-数据同步框架：Canal"><a href="#3-3-数据同步框架：Canal" class="headerlink" title="3.3 数据同步框架：Canal"></a>3.3 数据同步框架：Canal</h2><h3 id="3-3-1-Canal简介"><a href="#3-3-1-Canal简介" class="headerlink" title="3.3.1 Canal简介"></a>3.3.1 Canal简介</h3><p>阿里巴巴MySQL数据库binlog增量订阅与消费组件，基于数据库增量日志解析，提供增量数据订阅与消费，目前主要支持了MySQL。</p>
<p>Canal开源地址：<a target="_blank" rel="noopener" href="https://github.com/alibaba/canal%E3%80%82">https://github.com/alibaba/canal。</a></p>
<h3 id="3-3-2-Canal工作原理"><a href="#3-3-2-Canal工作原理" class="headerlink" title="3.3.2 Canal工作原理"></a>3.3.2 Canal工作原理</h3><p><strong>MySQL主从复制的实现</strong></p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110656500.png" alt="img"></p>
<p>从上图可以看出，主从复制主要分成三步：</p>
<ul>
<li><p>Master节点将数据的改变记录到二进制日志（binary log）中（这些记录叫做二进制日志事件，binary log events，可以通过show binlog events进行查看）。</p>
</li>
<li><p>Slave节点将Master节点的二进制日志事件（binary log events）拷贝到它的中继日志（relay log）。</p>
</li>
<li><p>Slave节点重做中继日志中的事件将改变反映到自己本身的数据库中。</p>
</li>
</ul>
<h3 id="3-3-3-Canal内部原理"><a href="#3-3-3-Canal内部原理" class="headerlink" title="3.3.3 Canal内部原理"></a>3.3.3 Canal内部原理</h3><p>首先，我们来看下Canal的原理图，如下所示。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110711964.png" alt="img"></p>
<p>原理大致描述如下：</p>
<ul>
<li>Canal 模拟 MySQL slave 的交互协议，伪装自己为 MySQL Slave ，向 MySQL Master 发送dump 协议</li>
<li>MySQL Master 收到 dump 请求，开始推送 binary log 给 Slave (即 Canal )</li>
<li>Canal 解析 binary log 对象(原始为 byte 流)</li>
</ul>
<p><strong>优点</strong>：实时性、分布式、ACK机制</p>
<p><strong>缺点</strong>：</p>
<ul>
<li>只支持增量同步，不支持全量同步</li>
<li>MySQL -&gt; ES、RDB</li>
<li>一个 instance 只能有一个消费者</li>
<li>单点压力过大</li>
</ul>
<h3 id="3-3-4-Canal内部结构"><a href="#3-3-4-Canal内部结构" class="headerlink" title="3.3.4 Canal内部结构"></a>3.3.4 Canal内部结构</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110725934.png" alt="img"></p>
<p>说明如下：</p>
<ul>
<li>Server：代表一个Canal运行实例，对应一个JVM进程。</li>
<li>Instance：对应一个数据队列（1个Server对应1个或者多个Instance）。</li>
</ul>
<p>接下来，我们再来看下Instance下的子模块，如下所示。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/2020080611073643.png" alt="img"></p>
<ul>
<li>EventParser：数据源接入，模拟Slave协议和Master节点进行交互，协议解析。</li>
<li>EventSink：EventParser和EventStore的连接器，对数据进行过滤、加工、归并和分发等处理。</li>
<li>EventSore：数据存储。</li>
<li>MetaManager：增量订阅和消费信息管理。</li>
</ul>
<h3 id="3-3-5-Canal-环境准备"><a href="#3-3-5-Canal-环境准备" class="headerlink" title="3.3.5 Canal 环境准备"></a>3.3.5 Canal 环境准备</h3><h4 id="3-3-5-1-设置MySQL远程访问"><a href="#3-3-5-1-设置MySQL远程访问" class="headerlink" title="3.3.5.1 设置MySQL远程访问"></a>3.3.5.1 设置MySQL远程访问</h4><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">grant</span> <span class="keyword">all</span> privileges <span class="keyword">on</span> <span class="operator">*</span>.<span class="operator">*</span> <span class="keyword">to</span> <span class="string">&#x27;root&#x27;</span>@<span class="string">&#x27;%&#x27;</span> identified <span class="keyword">by</span> <span class="string">&#x27;123456&#x27;</span>;</span><br><span class="line">flush privileges;</span><br></pre></td></tr></table></figure>

<h4 id="3-3-5-2-MySQL配置"><a href="#3-3-5-2-MySQL配置" class="headerlink" title="3.3.5.2 MySQL配置"></a>3.3.5.2 MySQL配置</h4><p>注意：这里的MySQL是基于5.7版本进行说明的。</p>
<p>Canal的原理基于MySQL binlog技术，所以，要想使用Canal就要开启MySQL的binlog写入功能，建议配置binlog的模式为row。</p>
<p>可以在MySQL命令行输入如下命令来查看binlog的模式。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">SHOW</span> VARIABLES <span class="keyword">LIKE</span> <span class="string">&#x27;binlog_format&#x27;</span>;</span><br></pre></td></tr></table></figure>

<p>执行效果如下所示。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110746949.png" alt="img"></p>
<p>可以看到，在MySQL中默认的binlog格式为STATEMENT，这里我们需要将STATEMENT修改为ROW。修改/etc/my.cnf文件。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">vim /etc/my.cnf</span><br></pre></td></tr></table></figure>

<p>在[mysqld]下面新增如下三项配置。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">log-bin=mysql-bin  <span class="comment">#开启MySQL二进制日志</span></span><br><span class="line">binlog_format=ROW <span class="comment">#将二进制日志的格式设置为ROW</span></span><br><span class="line">server_id=1 <span class="comment">#server_id需要唯一，不能与Canal的slaveId重复</span></span><br></pre></td></tr></table></figure>

<p>修改完my.cnf文件后，需要重启MySQL服务。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">service mysqld restart</span><br></pre></td></tr></table></figure>

<p>接下来，我们再次查看binlog模式。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">SHOW</span> VARIABLES <span class="keyword">LIKE</span> <span class="string">&#x27;binlog_format&#x27;</span>;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110759401.png" alt="img"></p>
<p>可以看到，此时，MySQL的binlog模式已经被设置为ROW了。</p>
<h4 id="3-3-5-3-MySQL创建用户授权"><a href="#3-3-5-3-MySQL创建用户授权" class="headerlink" title="3.3.5.3 MySQL创建用户授权"></a>3.3.5.3 MySQL创建用户授权</h4><p>Canal的原理是模式自己为MySQL Slave，所以一定要设置MySQL Slave的相关权限。这里，需要创建一个主从同步的账户，并且赋予这个账户相关的权限。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">CREATE</span> <span class="keyword">USER</span> canal@<span class="string">&#x27;localhost&#x27;</span> IDENTIFIED <span class="keyword">BY</span> <span class="string">&#x27;canal&#x27;</span>;</span><br><span class="line"><span class="keyword">GRANT</span> <span class="keyword">SELECT</span>, REPLICATION SLAVE, REPLICATION CLIENT <span class="keyword">ON</span> <span class="operator">*</span>.<span class="operator">*</span> <span class="keyword">TO</span> <span class="string">&#x27;canal&#x27;</span>@<span class="string">&#x27;localhost&#x27;</span>;</span><br><span class="line">FLUSH PRIVILEGES;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110810452.png" alt="img"></p>
<h3 id="3-3-6-Canal部署安装"><a href="#3-3-6-Canal部署安装" class="headerlink" title="3.3.6 Canal部署安装"></a>3.3.6 Canal部署安装</h3><h4 id="3-3-6-1-下载Canal"><a href="#3-3-6-1-下载Canal" class="headerlink" title="3.3.6.1 下载Canal"></a>3.3.6.1 下载Canal</h4><p>这里，我们以Canal 1.1.1版本进行说明，小伙伴们可以到链接 <a target="_blank" rel="noopener" href="https://github.com/alibaba/canal/releases/tag/canal-1.1.1">https://github.com/alibaba/canal/releases/tag/canal-1.1.1</a> 下载Canal 1.1.1版本。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110822997.png" alt="img"></p>
<h4 id="3-3-6-2-上传解压"><a href="#3-3-6-2-上传解压" class="headerlink" title="3.3.6.2 上传解压"></a>3.3.6.2 上传解压</h4><p>将下载好的Canal安装包，上传到服务器，并执行如下命令进行解压</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">mkdir -p /usr/<span class="built_in">local</span>/canal</span><br><span class="line">tar -zxvf canal.deployer-1.1.1.tar.gz -C /usr/<span class="built_in">local</span>/canal/</span><br></pre></td></tr></table></figure>

<p>解压后的目录如下所示。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110834262.png" alt="img"></p>
<p>各目录的说明如下：</p>
<ul>
<li>bin：存储可执行脚本。</li>
<li>conf：存放配置文件。</li>
<li>lib：存放其他依赖或者第三方库。</li>
<li>logs：存放的是日志文件。</li>
</ul>
<h4 id="3-3-6-3-修改配置文件"><a href="#3-3-6-3-修改配置文件" class="headerlink" title="3.3.6.3 修改配置文件"></a>3.3.6.3 修改配置文件</h4><p>在Canal的conf目录下有一个canal.properties文件，这个文件中配置的是Canal Server相关的配置，在这个文件中有如下一行配置。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">canal.destinations=example</span><br></pre></td></tr></table></figure>

<p>这里的example就相当于Canal的一个Instance，可以在这里配置多个Instance，多个Instance之间以逗号分隔即可。同时，这里的example也对应着Canal的conf目录下的一个文件夹。也就是说，Canal中的每个Instance实例都对应着conf目录下的一个子目录。</p>
<p>接下来，我们需要修改Canal的conf目录下的example目录的一个配置文件instance.properties。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">vim instance.properties</span><br></pre></td></tr></table></figure>

<p>修改如下配置项。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#################################################</span></span><br><span class="line"><span class="comment">## canal slaveId,注意：不要与MySQL的server_id重复</span></span><br><span class="line">canal.instance.mysql.slaveId = 1234</span><br><span class="line"></span><br><span class="line"><span class="comment">#position info，需要改成自己的数据库信息</span></span><br><span class="line">canal.instance.master.address = 127.0.0.1:3306</span><br><span class="line">canal.instance.master.journal.name =</span><br><span class="line">canal.instance.master.position =</span><br><span class="line">canal.instance.master.timestamp =</span><br><span class="line"></span><br><span class="line"><span class="comment">#canal.instance.standby.address =</span></span><br><span class="line"><span class="comment">#canal.instance.standby.journal.name =</span></span><br><span class="line"><span class="comment">#canal.instance.standby.position =</span></span><br><span class="line"><span class="comment">#canal.instance.standby.timestamp =</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#username/password，需要改成自己的数据库信息</span></span><br><span class="line">canal.instance.dbUsername = canal</span><br><span class="line">canal.instance.dbPassword = canal</span><br><span class="line">canal.instance.defaultDatabaseName =canaldb</span><br><span class="line">canal.instance.connectionCharset = UTF-8</span><br><span class="line"></span><br><span class="line"><span class="comment">#table regex</span></span><br><span class="line">canal.instance.filter.regex = canaldb\\..*</span><br><span class="line"><span class="comment">#################################################</span></span><br></pre></td></tr></table></figure>

<p>选项含义：</p>
<ul>
<li>canal.instance.mysql.slaveId : mysql集群配置中的serverId概念，需要保证和当前mysql集群中id唯一;</li>
<li>canal.instance.master.address: mysql主库链接地址;</li>
<li>canal.instance.dbUsername : mysql数据库帐号;</li>
<li>canal.instance.dbPassword : mysql数据库密码;</li>
<li>canal.instance.defaultDatabaseName : mysql链接时默认数据库;</li>
<li>canal.instance.connectionCharset : mysql 数据解析编码;</li>
<li>canal.instance.filter.regex : mysql 数据解析关注的表，Perl正则表达式.</li>
</ul>
<h3 id="3-3-7-启动Canal"><a href="#3-3-7-启动Canal" class="headerlink" title="3.3.7 启动Canal"></a>3.3.7 启动Canal</h3><p>配置完Canal后，就可以启动Canal了。进入到Canal的bin目录下，输入如下命令启动Canal。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./startup.sh</span><br></pre></td></tr></table></figure>

<h3 id="3-3-8-测试Canal"><a href="#3-3-8-测试Canal" class="headerlink" title="3.3.8 测试Canal"></a>3.3.8 测试Canal</h3><h3 id="3-3-9-导入并修改源码"><a href="#3-3-9-导入并修改源码" class="headerlink" title="3.3.9 导入并修改源码"></a>3.3.9 导入并修改源码</h3><p>这里，我们使用Canal的源码进行测试，下载Canal的源码后，将其导入到IDEA中。</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/20200806110843888.png" alt="img"></p>
<p>接下来，我们找到example下的SimpleCanalClientTest类进行测试。这个类的源码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> com.alibaba.otter.canal.example;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.net.InetSocketAddress;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.client.CanalConnector;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.client.CanalConnectors;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.common.utils.AddressUtils;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 单机模式的测试例子</span></span><br><span class="line"><span class="comment"> * </span></span><br><span class="line"><span class="comment"> * <span class="doctag">@author</span> jianghang 2013-4-15 下午04:19:20</span></span><br><span class="line"><span class="comment"> * <span class="doctag">@version</span> 1.0.4</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">SimpleCanalClientTest</span> <span class="keyword">extends</span> <span class="title">AbstractCanalClientTest</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="title">SimpleCanalClientTest</span><span class="params">(String destination)</span></span>&#123;</span><br><span class="line">           <span class="keyword">super</span>(destination);</span><br><span class="line">     &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String args[])</span> </span>&#123;</span><br><span class="line">        <span class="comment">// 根据ip，直接创建链接，无HA的功能</span></span><br><span class="line">        String destination = <span class="string">&quot;example&quot;</span>;</span><br><span class="line">        String ip = AddressUtils.getHostIp();</span><br><span class="line">        CanalConnector connector = CanalConnectors.newSingleConnector(</span><br><span class="line">            <span class="keyword">new</span> InetSocketAddress(ip, <span class="number">11111</span>),</span><br><span class="line">                destination,</span><br><span class="line">                <span class="string">&quot;canal&quot;</span>,</span><br><span class="line">                <span class="string">&quot;canal&quot;</span>);</span><br><span class="line"></span><br><span class="line">        <span class="keyword">final</span> SimpleCanalClientTest clientTest = <span class="keyword">new</span> SimpleCanalClientTest(destination);</span><br><span class="line">        clientTest.setConnector(connector);</span><br><span class="line">        clientTest.start();</span><br><span class="line">        Runtime.getRuntime().addShutdownHook(<span class="keyword">new</span> Thread() &#123;</span><br><span class="line"></span><br><span class="line">            <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">run</span><span class="params">()</span> </span>&#123;</span><br><span class="line">                <span class="keyword">try</span> &#123;</span><br><span class="line">                    logger.info(<span class="string">&quot;## stop the canal client&quot;</span>);</span><br><span class="line">                    clientTest.stop();</span><br><span class="line">                &#125; <span class="keyword">catch</span> (Throwable e) &#123;</span><br><span class="line">                    logger.warn(<span class="string">&quot;##something goes wrong when stopping canal:&quot;</span>, e);</span><br><span class="line">                &#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">                    logger.info(<span class="string">&quot;## canal client is down.&quot;</span>);</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line"></span><br><span class="line">        &#125;);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>可以看到，这个类中，使用的destination为example。在这个类中，我们只需要将IP地址修改为Canal Server的IP即可。</p>
<p>具体为：将如下一行代码。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">String ip = AddressUtils.getHostIp();</span><br></pre></td></tr></table></figure>

<p>修改为：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">String ip = <span class="string">&quot;192.168.175.100&quot;</span></span><br></pre></td></tr></table></figure>

<p>由于我们在配置Canal时，没有指定用户名和密码，所以，我们还需要将如下代码。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">CanalConnector connector = CanalConnectors.newSingleConnector(</span><br><span class="line">    <span class="keyword">new</span> InetSocketAddress(ip, <span class="number">11111</span>),</span><br><span class="line">    destination,</span><br><span class="line">    <span class="string">&quot;canal&quot;</span>,</span><br><span class="line">    <span class="string">&quot;canal&quot;</span>);</span><br></pre></td></tr></table></figure>

<p>修改为：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">CanalConnector connector = CanalConnectors.newSingleConnector(</span><br><span class="line">    <span class="keyword">new</span> InetSocketAddress(ip, <span class="number">11111</span>),</span><br><span class="line">    destination,</span><br><span class="line">    <span class="string">&quot;&quot;</span>,</span><br><span class="line">    <span class="string">&quot;&quot;</span>);</span><br></pre></td></tr></table></figure>

<p>修改完成后，运行main方法启动程序。</p>
<h3 id="3-3-10-测试数据变更"><a href="#3-3-10-测试数据变更" class="headerlink" title="3.3.10 测试数据变更"></a>3.3.10 测试数据变更</h3><p>接下来，在MySQL中创建一个canaldb数据库。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">create</span> database canaldb;</span><br></pre></td></tr></table></figure>

<p>此时会在IDEA的命令行输出相关的日志信息。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">****************************************************</span><br><span class="line">* Batch Id: [7] ,count : [3] , memsize : [149] , Time : 2020-08-05 23:25:35</span><br><span class="line">* Start : [mysql-bin.000007:6180:1540286735000(2020-08-05 23:25:35)] </span><br><span class="line">* End : [mysql-bin.000007:6356:1540286735000(2020-08-05 23:25:35)] </span><br><span class="line">****************************************************</span><br></pre></td></tr></table></figure>

<p>接下来，我在canaldb数据库中创建数据表，并对数据表中的数据进行增删改查，程序输出的日志信息如下所示。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#在mysql进行数据变更后，这里会显示mysql的bin日志。</span></span><br><span class="line">****************************************************</span><br><span class="line">* Batch Id: [7] ,count : [3] , memsize : [149] , Time : 2020-08-05 23:25:35</span><br><span class="line">* Start : [mysql-bin.000007:6180:1540286735000(2020-08-05 23:25:35)] </span><br><span class="line">* End : [mysql-bin.000007:6356:1540286735000(2020-08-05 23:25:35)] </span><br><span class="line">****************************************************</span><br><span class="line"></span><br><span class="line">================&gt; binlog[mysql-bin.000007:6180] , executeTime : 1540286735000(2020-08-05 23:25:35) , gtid : () , delay : 393ms</span><br><span class="line"> BEGIN ----&gt; Thread id: 43</span><br><span class="line">----------------&gt; binlog[mysql-bin.000007:6311] , name[canal,canal_table] , eventType : DELETE , executeTime : 1540286735000(2020-08-05 23:25:35) , gtid : () , delay : 393 ms</span><br><span class="line">id : 8    <span class="built_in">type</span>=int(10) unsigned</span><br><span class="line">name : 512    <span class="built_in">type</span>=varchar(255)</span><br><span class="line">----------------</span><br><span class="line"> END ----&gt; transaction id: 249</span><br><span class="line">================&gt; binlog[mysql-bin.000007:6356] , executeTime : 1540286735000(2020-08-05 23:25:35) , gtid : () , delay : 394ms</span><br><span class="line"></span><br><span class="line">****************************************************</span><br><span class="line">* Batch Id: [8] ,count : [3] , memsize : [149] , Time : 2020-08-05 23:25:35</span><br><span class="line">* Start : [mysql-bin.000007:6387:1540286869000(2020-08-05 23:25:49)] </span><br><span class="line">* End : [mysql-bin.000007:6563:1540286869000(2020-08-05 23:25:49)] </span><br><span class="line">****************************************************</span><br><span class="line"></span><br><span class="line">================&gt; binlog[mysql-bin.000007:6387] , executeTime : 1540286869000(2020-08-05 23:25:49) , gtid : () , delay : 976ms</span><br><span class="line"> BEGIN ----&gt; Thread id: 43</span><br><span class="line">----------------&gt; binlog[mysql-bin.000007:6518] , name[canal,canal_table] , eventType : INSERT , executeTime : 1540286869000(2020-08-05 23:25:49) , gtid : () , delay : 976 ms</span><br><span class="line">id : 21    <span class="built_in">type</span>=int(10) unsigned    update=<span class="literal">true</span></span><br><span class="line">name : aaa    <span class="built_in">type</span>=varchar(255)    update=<span class="literal">true</span></span><br><span class="line">----------------</span><br><span class="line"> END ----&gt; transaction id: 250</span><br><span class="line">================&gt; binlog[mysql-bin.000007:6563] , executeTime : 1540286869000(2020-08-05 23:25:49) , gtid : () , delay : 977ms</span><br><span class="line"></span><br><span class="line">****************************************************</span><br><span class="line">* Batch Id: [9] ,count : [3] , memsize : [161] , Time : 2020-08-05 23:26:22</span><br><span class="line">* Start : [mysql-bin.000007:6594:1540286902000(2020-08-05 23:26:22)] </span><br><span class="line">* End : [mysql-bin.000007:6782:1540286902000(2020-08-05 23:26:22)] </span><br><span class="line">****************************************************</span><br><span class="line"></span><br><span class="line">================&gt; binlog[mysql-bin.000007:6594] , executeTime : 1540286902000(2020-08-05 23:26:22) , gtid : () , delay : 712ms</span><br><span class="line"> BEGIN ----&gt; Thread id: 43</span><br><span class="line">----------------&gt; binlog[mysql-bin.000007:6725] , name[canal,canal_table] , eventType : UPDATE , executeTime : 1540286902000(2020-08-05 23:26:22) , gtid : () , delay : 712 ms</span><br><span class="line">id : 21    <span class="built_in">type</span>=int(10) unsigned</span><br><span class="line">name : aaac    <span class="built_in">type</span>=varchar(255)    update=<span class="literal">true</span></span><br><span class="line">----------------</span><br><span class="line"> END ----&gt; transaction id: 252</span><br><span class="line">================&gt; binlog[mysql-bin.000007:6782] , executeTime : 1540286902000(2020-08-05 23:26:22) , gtid : () , delay : 713ms</span><br></pre></td></tr></table></figure>

<h3 id="3-3-11-数据同步实现"><a href="#3-3-11-数据同步实现" class="headerlink" title="3.3.11 数据同步实现"></a>3.3.11 数据同步实现</h3><p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210426230029824.png" alt="image-20210426230029824"></p>
<h4 id="3-3-11-1-需求"><a href="#3-3-11-1-需求" class="headerlink" title="3.3.11.1 需求"></a>3.3.11.1 需求</h4><p>将数据库数据的变化, 通过canal解析binlog日志, 实时更新到solr(ES 也可以)的索引库中。</p>
<h4 id="3-3-11-2-具体实现"><a href="#3-3-11-2-具体实现" class="headerlink" title="3.3.11.2 具体实现"></a>3.3.11.2 具体实现</h4><p><strong>创建工程</strong></p>
<p>创建Maven工程mykit-canal-demo，并在pom.xml文件中添加如下配置。</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependencies</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>com.alibaba.otter<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>canal.client<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>1.0.24<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>com.alibaba.otter<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>canal.protocol<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>1.0.24<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>commons-lang<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>commons-lang<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>2.6<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.codehaus.jackson<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>jackson-mapper-asl<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>1.8.9<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.apache.solr<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>solr-solrj<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>4.10.3<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>junit<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>junit<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>4.9<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">scope</span>&gt;</span>test<span class="tag">&lt;/<span class="name">scope</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependencies</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p><strong>创建log4j配置文件</strong></p>
<p>在工程的src/main/resources目录下创建log4j.properties文件，内容如下所示。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">log4j.rootCategory=debug, CONSOLE</span><br><span class="line"></span><br><span class="line"><span class="comment"># CONSOLE is set to be a ConsoleAppender using a PatternLayout.</span></span><br><span class="line">log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender</span><br><span class="line">log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout</span><br><span class="line">log4j.appender.CONSOLE.layout.ConversionPattern=%d&#123;ISO8601&#125; %-6r [%15.15t] %-5p %30.30c %x - %m\n</span><br><span class="line"></span><br><span class="line"><span class="comment"># LOGFILE is set to be a File appender using a PatternLayout.</span></span><br><span class="line"><span class="comment"># log4j.appender.LOGFILE=org.apache.log4j.FileAppender</span></span><br><span class="line"><span class="comment"># log4j.appender.LOGFILE.File=d:\axis.log</span></span><br><span class="line"><span class="comment"># log4j.appender.LOGFILE.Append=true</span></span><br><span class="line"><span class="comment"># log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout</span></span><br><span class="line"><span class="comment"># log4j.appender.LOGFILE.layout.ConversionPattern=%d&#123;ISO8601&#125; %-6r [%15.15t] %-5p %30.30c %x - %m\n</span></span><br></pre></td></tr></table></figure>

<p><strong>创建实体类</strong></p>
<p>在io.mykit.canal.demo.bean包下创建一个Book实体类，用于测试Canal的数据传输，如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.bean;</span><br><span class="line"><span class="keyword">import</span> org.apache.solr.client.solrj.beans.Field;</span><br><span class="line"><span class="keyword">import</span> java.util.Date;</span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">Book</span> <span class="keyword">implements</span> <span class="title">Serializable</span> </span>&#123;</span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> <span class="keyword">long</span> serialVersionUID = -<span class="number">6350345408771427834L</span>;&#123;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;id&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> Integer id;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;book_name&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> String name;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;book_author&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> String author;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;book_publishtime&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> Date publishtime;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;book_price&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> Double price;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Field(&quot;book_publishgroup&quot;)</span></span><br><span class="line">    <span class="keyword">private</span> String publishgroup;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Integer <span class="title">getId</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> id;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setId</span><span class="params">(Integer id)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.id = id;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> String <span class="title">getName</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> name;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setName</span><span class="params">(String name)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.name = name;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> String <span class="title">getAuthor</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> author;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setAuthor</span><span class="params">(String author)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.author = author;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Date <span class="title">getPublishtime</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> publishtime;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setPublishtime</span><span class="params">(Date publishtime)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.publishtime = publishtime;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Double <span class="title">getPrice</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> price;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setPrice</span><span class="params">(Double price)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.price = price;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> String <span class="title">getPublishgroup</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> publishgroup;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setPublishgroup</span><span class="params">(String publishgroup)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">this</span>.publishgroup = publishgroup;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> String <span class="title">toString</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&quot;Book&#123;&quot;</span> +</span><br><span class="line">                <span class="string">&quot;id=&quot;</span> + id +</span><br><span class="line">                <span class="string">&quot;, name=&#x27;&quot;</span> + name + <span class="string">&#x27;\&#x27;&#x27;</span> +</span><br><span class="line">                <span class="string">&quot;, author=&#x27;&quot;</span> + author + <span class="string">&#x27;\&#x27;&#x27;</span> +</span><br><span class="line">                <span class="string">&quot;, publishtime=&quot;</span> + publishtime +</span><br><span class="line">                <span class="string">&quot;, price=&quot;</span> + price +</span><br><span class="line">                <span class="string">&quot;, publishgroup=&#x27;&quot;</span> + publishgroup + <span class="string">&#x27;\&#x27;&#x27;</span> +</span><br><span class="line">                <span class="string">&#x27;&#125;&#x27;</span>;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>其中，我们在Book实体类中，使用Solr的注解@Field定义了实体类字段与Solr域之间的关系。</p>
<p><strong>各种工具类的实现</strong></p>
<p>接下来，我们就在io.mykit.canal.demo.utils包下创建各种工具类。</p>
<ul>
<li>BinlogValue</li>
</ul>
<p>用于存储binlog分析的每行每列的value值，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.utils;</span><br><span class="line"><span class="keyword">import</span> java.io.Serializable;</span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * </span></span><br><span class="line"><span class="comment"> * ClassName: BinlogValue &lt;br/&gt; </span></span><br><span class="line"><span class="comment"> * </span></span><br><span class="line"><span class="comment"> * binlog分析的每行每列的value值；&lt;br&gt;</span></span><br><span class="line"><span class="comment"> * 新增数据：beforeValue 和 value 均为现有值；&lt;br&gt;</span></span><br><span class="line"><span class="comment"> * 修改数据：beforeValue是修改前的值；value为修改后的值；&lt;br&gt;</span></span><br><span class="line"><span class="comment"> * 删除数据：beforeValue和value均是删除前的值； 这个比较特殊主要是为了删除数据时方便获取删除前的值&lt;br&gt;</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">BinlogValue</span> <span class="keyword">implements</span> <span class="title">Serializable</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> <span class="keyword">long</span> serialVersionUID = -<span class="number">6350345408773943086L</span>;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">private</span> String value;</span><br><span class="line">	<span class="keyword">private</span> String beforeValue;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * binlog分析的每行每列的value值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 新增数据： value：为现有值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 修改数据：value为修改后的值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 删除数据：value是删除前的值； 这个比较特殊主要是为了删除数据时方便获取删除前的值&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> String <span class="title">getValue</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> value;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setValue</span><span class="params">(String value)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.value = value;</span><br><span class="line">	&#125;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * binlog分析的每行每列的beforeValue值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 新增数据：beforeValue为现有值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 修改数据：beforeValue是修改前的值；&lt;br&gt;</span></span><br><span class="line"><span class="comment">	 * 删除数据：beforeValue为删除前的值； &lt;br&gt;</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> String <span class="title">getBeforeValue</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> beforeValue;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setBeforeValue</span><span class="params">(String beforeValue)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.beforeValue = beforeValue;</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li>CanalDataParser</li>
</ul>
<p>用于解析数据，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.utils;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.text.SimpleDateFormat;</span><br><span class="line"><span class="keyword">import</span> java.util.ArrayList;</span><br><span class="line"><span class="keyword">import</span> java.util.Date;</span><br><span class="line"><span class="keyword">import</span> java.util.HashMap;</span><br><span class="line"><span class="keyword">import</span> java.util.List;</span><br><span class="line"><span class="keyword">import</span> java.util.Map;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.commons.lang.SystemUtils;</span><br><span class="line"><span class="keyword">import</span> org.slf4j.Logger;</span><br><span class="line"><span class="keyword">import</span> org.slf4j.LoggerFactory;</span><br><span class="line"><span class="keyword">import</span> org.springframework.util.CollectionUtils;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.Message;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.Column;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.Entry;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.EntryType;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.EventType;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.RowChange;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.RowData;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd;</span><br><span class="line"><span class="keyword">import</span> com.google.protobuf.InvalidProtocolBufferException;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 解析数据</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">CanalDataParser</span> </span>&#123;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">protected</span> <span class="keyword">static</span> <span class="keyword">final</span> String DATE_FORMAT 	= <span class="string">&quot;yyyy-MM-dd HH:mm:ss&quot;</span>;</span><br><span class="line">	<span class="keyword">protected</span> <span class="keyword">static</span> <span class="keyword">final</span> String yyyyMMddHHmmss = <span class="string">&quot;yyyyMMddHHmmss&quot;</span>;</span><br><span class="line">	<span class="keyword">protected</span> <span class="keyword">static</span> <span class="keyword">final</span> String yyyyMMdd 		= <span class="string">&quot;yyyyMMdd&quot;</span>;</span><br><span class="line">	<span class="keyword">protected</span> <span class="keyword">static</span> <span class="keyword">final</span> String SEP 			= SystemUtils.LINE_SEPARATOR;</span><br><span class="line">	<span class="keyword">protected</span> <span class="keyword">static</span> String  context_format     = <span class="keyword">null</span>;</span><br><span class="line">    <span class="keyword">protected</span> <span class="keyword">static</span> String  row_format         = <span class="keyword">null</span>;</span><br><span class="line">    <span class="keyword">protected</span> <span class="keyword">static</span> String  transaction_format = <span class="keyword">null</span>;</span><br><span class="line">    <span class="keyword">protected</span> <span class="keyword">static</span> String row_log = <span class="keyword">null</span>;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> Logger logger = LoggerFactory.getLogger(CanalDataParser.class);</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">static</span> &#123;</span><br><span class="line">        context_format = SEP + <span class="string">&quot;****************************************************&quot;</span> + SEP;</span><br><span class="line">        context_format += <span class="string">&quot;* Batch Id: [&#123;&#125;] ,count : [&#123;&#125;] , memsize : [&#123;&#125;] , Time : &#123;&#125;&quot;</span> + SEP;</span><br><span class="line">        context_format += <span class="string">&quot;* Start : [&#123;&#125;] &quot;</span> + SEP;</span><br><span class="line">        context_format += <span class="string">&quot;* End : [&#123;&#125;] &quot;</span> + SEP;</span><br><span class="line">        context_format += <span class="string">&quot;****************************************************&quot;</span> + SEP;</span><br><span class="line"></span><br><span class="line">        row_format = SEP</span><br><span class="line">                     + <span class="string">&quot;----------------&gt; binlog[&#123;&#125;:&#123;&#125;] , name[&#123;&#125;,&#123;&#125;] , eventType : &#123;&#125; , executeTime : &#123;&#125; , delay : &#123;&#125;ms&quot;</span></span><br><span class="line">                     + SEP;</span><br><span class="line"></span><br><span class="line">        transaction_format = SEP + <span class="string">&quot;================&gt; binlog[&#123;&#125;:&#123;&#125;] , executeTime : &#123;&#125; , delay : &#123;&#125;ms&quot;</span> + SEP;</span><br><span class="line"></span><br><span class="line">        row_log = <span class="string">&quot;schema[&#123;&#125;], table[&#123;&#125;]&quot;</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">static</span> List&lt;InnerBinlogEntry&gt; <span class="title">convertToInnerBinlogEntry</span><span class="params">(Message message)</span> </span>&#123;</span><br><span class="line">		List&lt;InnerBinlogEntry&gt; innerBinlogEntryList = <span class="keyword">new</span> ArrayList&lt;InnerBinlogEntry&gt;();</span><br><span class="line">		</span><br><span class="line">		<span class="keyword">if</span>(message == <span class="keyword">null</span>) &#123;</span><br><span class="line">			logger.info(<span class="string">&quot;接收到空的 message; 忽略&quot;</span>);</span><br><span class="line">			<span class="keyword">return</span> innerBinlogEntryList;</span><br><span class="line">		&#125;</span><br><span class="line">		</span><br><span class="line">		<span class="keyword">long</span> batchId = message.getId();</span><br><span class="line">        <span class="keyword">int</span> size = message.getEntries().size();</span><br><span class="line">        <span class="keyword">if</span> (batchId == -<span class="number">1</span> || size == <span class="number">0</span>) &#123;</span><br><span class="line">        	logger.info(<span class="string">&quot;接收到空的message[size=&quot;</span> + size + <span class="string">&quot;]; 忽略&quot;</span>);</span><br><span class="line">        	<span class="keyword">return</span> innerBinlogEntryList;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        printLog(message, batchId, size);</span><br><span class="line">        List&lt;Entry&gt; entrys = message.getEntries();</span><br><span class="line"></span><br><span class="line">        <span class="comment">//输出日志</span></span><br><span class="line">        <span class="keyword">for</span> (Entry entry : entrys) &#123;</span><br><span class="line">        	<span class="keyword">long</span> executeTime = entry.getHeader().getExecuteTime();</span><br><span class="line">            <span class="keyword">long</span> delayTime = <span class="keyword">new</span> Date().getTime() - executeTime;</span><br><span class="line">        	</span><br><span class="line">            <span class="keyword">if</span> (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) &#123;</span><br><span class="line">                <span class="keyword">if</span> (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) &#123;</span><br><span class="line">                    TransactionBegin begin = <span class="keyword">null</span>;</span><br><span class="line">                    <span class="keyword">try</span> &#123;</span><br><span class="line">                        begin = TransactionBegin.parseFrom(entry.getStoreValue());</span><br><span class="line">                    &#125; <span class="keyword">catch</span> (InvalidProtocolBufferException e) &#123;</span><br><span class="line">                        <span class="keyword">throw</span> <span class="keyword">new</span> RuntimeException(<span class="string">&quot;parse event has an error , data:&quot;</span> + entry.toString(), e);</span><br><span class="line">                    &#125;</span><br><span class="line">                    <span class="comment">// 打印事务头信息，执行的线程id，事务耗时</span></span><br><span class="line">                    logger.info(<span class="string">&quot;BEGIN ----&gt; Thread id: &#123;&#125;&quot;</span>,  begin.getThreadId());</span><br><span class="line">                    logger.info(transaction_format, <span class="keyword">new</span> Object[] &#123;entry.getHeader().getLogfileName(),</span><br><span class="line">                                String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) &#125;);</span><br><span class="line"></span><br><span class="line">                &#125; <span class="keyword">else</span> <span class="keyword">if</span> (entry.getEntryType() == EntryType.TRANSACTIONEND) &#123;</span><br><span class="line">                    TransactionEnd end = <span class="keyword">null</span>;</span><br><span class="line">                    <span class="keyword">try</span> &#123;</span><br><span class="line">                        end = TransactionEnd.parseFrom(entry.getStoreValue());</span><br><span class="line">                    &#125; <span class="keyword">catch</span> (InvalidProtocolBufferException e) &#123;</span><br><span class="line">                        <span class="keyword">throw</span> <span class="keyword">new</span> RuntimeException(<span class="string">&quot;parse event has an error , data:&quot;</span> + entry.toString(), e);</span><br><span class="line">                    &#125;</span><br><span class="line">                    <span class="comment">// 打印事务提交信息，事务id</span></span><br><span class="line">                    logger.info(<span class="string">&quot;END ----&gt; transaction id: &#123;&#125;&quot;</span>, end.getTransactionId());</span><br><span class="line">                    logger.info(transaction_format,</span><br><span class="line">                        <span class="keyword">new</span> Object[] &#123;entry.getHeader().getLogfileName(),  String.valueOf(entry.getHeader().getLogfileOffset()),</span><br><span class="line">                                String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) &#125;);</span><br><span class="line">                &#125;</span><br><span class="line">                <span class="keyword">continue</span>;</span><br><span class="line">            &#125;</span><br><span class="line"></span><br><span class="line">            <span class="comment">//解析结果</span></span><br><span class="line">            <span class="keyword">if</span> (entry.getEntryType() == EntryType.ROWDATA) &#123;</span><br><span class="line">                RowChange rowChage = <span class="keyword">null</span>;</span><br><span class="line">                <span class="keyword">try</span> &#123;</span><br><span class="line">                    rowChage = RowChange.parseFrom(entry.getStoreValue());</span><br><span class="line">                &#125; <span class="keyword">catch</span> (Exception e) &#123;</span><br><span class="line">                    <span class="keyword">throw</span> <span class="keyword">new</span> RuntimeException(<span class="string">&quot;parse event has an error , data:&quot;</span> + entry.toString(), e);</span><br><span class="line">                &#125;</span><br><span class="line"></span><br><span class="line">                EventType eventType = rowChage.getEventType();</span><br><span class="line"></span><br><span class="line">                logger.info(row_format, <span class="keyword">new</span> Object[] &#123; entry.getHeader().getLogfileName(),</span><br><span class="line">                            String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(),</span><br><span class="line">                            entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) &#125;);</span><br><span class="line"></span><br><span class="line">                <span class="comment">//组装数据结果</span></span><br><span class="line">                <span class="keyword">if</span> (eventType == EventType.INSERT || eventType == EventType.DELETE || eventType == EventType.UPDATE) &#123;</span><br><span class="line">                	String schemaName = entry.getHeader().getSchemaName();</span><br><span class="line">                	String tableName = entry.getHeader().getTableName();</span><br><span class="line">                	List&lt;Map&lt;String, BinlogValue&gt;&gt; rows = parseEntry(entry);</span><br><span class="line"></span><br><span class="line">                	InnerBinlogEntry innerBinlogEntry = <span class="keyword">new</span> InnerBinlogEntry();</span><br><span class="line">                	innerBinlogEntry.setEntry(entry);</span><br><span class="line">                	innerBinlogEntry.setEventType(eventType);</span><br><span class="line">                	innerBinlogEntry.setSchemaName(schemaName);</span><br><span class="line">                	innerBinlogEntry.setTableName(tableName.toLowerCase());</span><br><span class="line">                	innerBinlogEntry.setRows(rows);</span><br><span class="line"></span><br><span class="line">                	innerBinlogEntryList.add(innerBinlogEntry);</span><br><span class="line">                &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">                	logger.info(<span class="string">&quot; 存在 INSERT INSERT UPDATE 操作之外的SQL [&quot;</span> + eventType.toString() + <span class="string">&quot;]&quot;</span>);</span><br><span class="line">                &#125;</span><br><span class="line">                <span class="keyword">continue</span>;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">		<span class="keyword">return</span> innerBinlogEntryList;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> List&lt;Map&lt;String, BinlogValue&gt;&gt; parseEntry(Entry entry) &#123;</span><br><span class="line">		List&lt;Map&lt;String, BinlogValue&gt;&gt; rows = <span class="keyword">new</span> ArrayList&lt;Map&lt;String, BinlogValue&gt;&gt;();</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			String schemaName = entry.getHeader().getSchemaName();</span><br><span class="line">        	String tableName = entry.getHeader().getTableName();</span><br><span class="line">			RowChange rowChage = RowChange.parseFrom(entry.getStoreValue());</span><br><span class="line">			EventType eventType = rowChage.getEventType();</span><br><span class="line"></span><br><span class="line">			<span class="comment">// 处理每个Entry中的每行数据</span></span><br><span class="line">			<span class="keyword">for</span> (RowData rowData : rowChage.getRowDatasList()) &#123;</span><br><span class="line">				StringBuilder rowlog = <span class="keyword">new</span> StringBuilder(<span class="string">&quot;rowlog schema[&quot;</span> + schemaName + <span class="string">&quot;], table[&quot;</span> + tableName + <span class="string">&quot;], event[&quot;</span> + eventType.toString() + <span class="string">&quot;]&quot;</span>);</span><br><span class="line">				</span><br><span class="line">				Map&lt;String, BinlogValue&gt; row = <span class="keyword">new</span> HashMap&lt;String, BinlogValue&gt;();</span><br><span class="line">				List&lt;Column&gt; beforeColumns = rowData.getBeforeColumnsList();</span><br><span class="line">				List&lt;Column&gt; afterColumns = rowData.getAfterColumnsList();</span><br><span class="line">				beforeColumns = rowData.getBeforeColumnsList();</span><br><span class="line">			    <span class="keyword">if</span> (eventType == EventType.DELETE) &#123;<span class="comment">//delete</span></span><br><span class="line">			    	<span class="keyword">for</span>(Column column : beforeColumns) &#123;</span><br><span class="line">			    		BinlogValue binlogValue = <span class="keyword">new</span> BinlogValue();</span><br><span class="line">			    		binlogValue.setValue(column.getValue());</span><br><span class="line">			    		binlogValue.setBeforeValue(column.getValue());</span><br><span class="line">				    	row.put(column.getName(), binlogValue);</span><br><span class="line">				    &#125;</span><br><span class="line">			    &#125; <span class="keyword">else</span> <span class="keyword">if</span>(eventType == EventType.UPDATE) &#123;<span class="comment">//update</span></span><br><span class="line">			    	<span class="keyword">for</span>(Column column : beforeColumns) &#123;</span><br><span class="line">			    		BinlogValue binlogValue = <span class="keyword">new</span> BinlogValue();</span><br><span class="line">			    		binlogValue.setBeforeValue(column.getValue());</span><br><span class="line">				    	row.put(column.getName(), binlogValue);</span><br><span class="line">				    &#125;</span><br><span class="line">			    	<span class="keyword">for</span>(Column column : afterColumns) &#123;</span><br><span class="line">			    		BinlogValue binlogValue = row.get(column.getName());</span><br><span class="line">			    		<span class="keyword">if</span>(binlogValue == <span class="keyword">null</span>) &#123;</span><br><span class="line">			    			binlogValue = <span class="keyword">new</span> BinlogValue();</span><br><span class="line">			    		&#125;</span><br><span class="line">			    		binlogValue.setValue(column.getValue());</span><br><span class="line">				    	row.put(column.getName(), binlogValue);</span><br><span class="line">				    &#125;</span><br><span class="line">			    &#125; <span class="keyword">else</span> &#123; <span class="comment">// insert</span></span><br><span class="line">			    	<span class="keyword">for</span>(Column column : afterColumns) &#123;</span><br><span class="line">			    		BinlogValue binlogValue = <span class="keyword">new</span> BinlogValue();</span><br><span class="line">			    		binlogValue.setValue(column.getValue());</span><br><span class="line">			    		binlogValue.setBeforeValue(column.getValue());</span><br><span class="line">				    	row.put(column.getName(), binlogValue);</span><br><span class="line">				    &#125;</span><br><span class="line">			    &#125; </span><br><span class="line">			   </span><br><span class="line">			    rows.add(row);</span><br><span class="line">			    String rowjson = JacksonUtil.obj2str(row);</span><br><span class="line">			    </span><br><span class="line">			    logger.info(<span class="string">&quot;########################### Data Parse Result ###########################&quot;</span>);</span><br><span class="line">			    logger.info(rowlog + <span class="string">&quot; , &quot;</span> + rowjson);</span><br><span class="line">			    logger.info(<span class="string">&quot;########################### Data Parse Result ###########################&quot;</span>);</span><br><span class="line">			    logger.info(<span class="string">&quot;&quot;</span>);</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">catch</span> (InvalidProtocolBufferException e) &#123;</span><br><span class="line">			<span class="keyword">throw</span> <span class="keyword">new</span> RuntimeException(<span class="string">&quot;parseEntry has an error , data:&quot;</span> + entry.toString(), e);</span><br><span class="line">		&#125;</span><br><span class="line">        <span class="keyword">return</span> rows;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="function"><span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">printLog</span><span class="params">(Message message, <span class="keyword">long</span> batchId, <span class="keyword">int</span> size)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">long</span> memsize = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span> (Entry entry : message.getEntries()) &#123;</span><br><span class="line">            memsize += entry.getHeader().getEventLength();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        String startPosition = <span class="keyword">null</span>;</span><br><span class="line">        String endPosition = <span class="keyword">null</span>;</span><br><span class="line">        <span class="keyword">if</span> (!CollectionUtils.isEmpty(message.getEntries())) &#123;</span><br><span class="line">            startPosition = buildPositionForDump(message.getEntries().get(<span class="number">0</span>));</span><br><span class="line">            endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - <span class="number">1</span>));</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        SimpleDateFormat format = <span class="keyword">new</span> SimpleDateFormat(DATE_FORMAT);</span><br><span class="line">        logger.info(context_format, <span class="keyword">new</span> Object[] &#123;batchId, size, memsize, format.format(<span class="keyword">new</span> Date()), startPosition, endPosition &#125;);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">	<span class="function"><span class="keyword">private</span> <span class="keyword">static</span> String <span class="title">buildPositionForDump</span><span class="params">(Entry entry)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">long</span> time = entry.getHeader().getExecuteTime();</span><br><span class="line">        Date date = <span class="keyword">new</span> Date(time);</span><br><span class="line">        SimpleDateFormat format = <span class="keyword">new</span> SimpleDateFormat(DATE_FORMAT);</span><br><span class="line">        <span class="keyword">return</span> entry.getHeader().getLogfileName() + <span class="string">&quot;:&quot;</span> + entry.getHeader().getLogfileOffset() + <span class="string">&quot;:&quot;</span> + entry.getHeader().getExecuteTime() + <span class="string">&quot;(&quot;</span> + format.format(date) + <span class="string">&quot;)&quot;</span>;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li>DateUtils</li>
</ul>
<p>时间工具类，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.utils;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.text.ParseException;</span><br><span class="line"><span class="keyword">import</span> java.text.SimpleDateFormat;</span><br><span class="line"><span class="keyword">import</span> java.util.Date;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">DateUtils</span> </span>&#123;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> String FORMAT_PATTERN = <span class="string">&quot;yyyy-MM-dd HH:mm:ss&quot;</span>;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">private</span> <span class="keyword">static</span> SimpleDateFormat sdf = <span class="keyword">new</span> SimpleDateFormat(FORMAT_PATTERN);</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">static</span> Date <span class="title">parseDate</span><span class="params">(String datetime)</span> <span class="keyword">throws</span> ParseException</span>&#123;</span><br><span class="line">		<span class="keyword">if</span>(datetime != <span class="keyword">null</span> &amp;&amp; !<span class="string">&quot;&quot;</span>.equals(datetime))&#123;</span><br><span class="line">			<span class="keyword">return</span> sdf.parse(datetime);</span><br><span class="line">		&#125;</span><br><span class="line">		<span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">	&#125;</span><br><span class="line">	</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">static</span> String <span class="title">formatDate</span><span class="params">(Date datetime)</span> <span class="keyword">throws</span> ParseException</span>&#123;</span><br><span class="line">		<span class="keyword">if</span>(datetime != <span class="keyword">null</span> )&#123;</span><br><span class="line">			<span class="keyword">return</span> sdf.format(datetime);</span><br><span class="line">		&#125;</span><br><span class="line">		<span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">	&#125;</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">static</span> Long <span class="title">formatStringDateToLong</span><span class="params">(String datetime)</span> <span class="keyword">throws</span> ParseException</span>&#123;</span><br><span class="line">		<span class="keyword">if</span>(datetime != <span class="keyword">null</span> &amp;&amp; !<span class="string">&quot;&quot;</span>.equals(datetime))&#123;</span><br><span class="line">			Date d =  sdf.parse(datetime);</span><br><span class="line">			<span class="keyword">return</span> d.getTime();</span><br><span class="line">		&#125;</span><br><span class="line">		<span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">	&#125;</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">static</span> Long <span class="title">formatDateToLong</span><span class="params">(Date datetime)</span> <span class="keyword">throws</span> ParseException</span>&#123;</span><br><span class="line">		<span class="keyword">if</span>(datetime != <span class="keyword">null</span>)&#123;</span><br><span class="line">			<span class="keyword">return</span> datetime.getTime();</span><br><span class="line">		&#125;</span><br><span class="line">		<span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li>InnerBinlogEntry</li>
</ul>
<p>Binlog实体类，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.utils;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.ArrayList;</span><br><span class="line"><span class="keyword">import</span> java.util.List;</span><br><span class="line"><span class="keyword">import</span> java.util.Map;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.Entry;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry.EventType;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">InnerBinlogEntry</span> </span>&#123;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * canal原生的Entry</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="keyword">private</span> Entry entry;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 该Entry归属于的表名</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="keyword">private</span> String tableName;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 该Entry归属数据库名</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="keyword">private</span> String schemaName;</span><br><span class="line">	</span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 该Entry本次的操作类型，对应canal原生的枚举；EventType.INSERT; EventType.UPDATE; EventType.DELETE;</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="keyword">private</span> EventType eventType;</span><br><span class="line">	</span><br><span class="line">	<span class="keyword">private</span> List&lt;Map&lt;String, BinlogValue&gt;&gt; rows = <span class="keyword">new</span> ArrayList&lt;Map&lt;String, BinlogValue&gt;&gt;();</span><br><span class="line">	</span><br><span class="line">	</span><br><span class="line">	<span class="function"><span class="keyword">public</span> Entry <span class="title">getEntry</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> entry;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setEntry</span><span class="params">(Entry entry)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.entry = entry;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> String <span class="title">getTableName</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> tableName;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setTableName</span><span class="params">(String tableName)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.tableName = tableName;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> EventType <span class="title">getEventType</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> eventType;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setEventType</span><span class="params">(EventType eventType)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.eventType = eventType;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> String <span class="title">getSchemaName</span><span class="params">()</span> </span>&#123;</span><br><span class="line">		<span class="keyword">return</span> schemaName;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setSchemaName</span><span class="params">(String schemaName)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.schemaName = schemaName;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="keyword">public</span> List&lt;Map&lt;String, BinlogValue&gt;&gt; getRows() &#123;</span><br><span class="line">		<span class="keyword">return</span> rows;</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">setRows</span><span class="params">(List&lt;Map&lt;String, BinlogValue&gt;&gt; rows)</span> </span>&#123;</span><br><span class="line">		<span class="keyword">this</span>.rows = rows;</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li>JacksonUtil</li>
</ul>
<p>Json工具类，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.utils;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.io.IOException;</span><br><span class="line"><span class="keyword">import</span> org.codehaus.jackson.JsonGenerationException;</span><br><span class="line"><span class="keyword">import</span> org.codehaus.jackson.JsonParseException;</span><br><span class="line"><span class="keyword">import</span> org.codehaus.jackson.map.JsonMappingException;</span><br><span class="line"><span class="keyword">import</span> org.codehaus.jackson.map.ObjectMapper;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">JacksonUtil</span> </span>&#123;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">static</span> ObjectMapper mapper = <span class="keyword">new</span> ObjectMapper();</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> String <span class="title">obj2str</span><span class="params">(Object obj)</span> </span>&#123;</span><br><span class="line">        String json = <span class="keyword">null</span>;</span><br><span class="line">        <span class="keyword">try</span> &#123;</span><br><span class="line">            json = mapper.writeValueAsString(obj);</span><br><span class="line">        &#125; <span class="keyword">catch</span> (JsonGenerationException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125; <span class="keyword">catch</span> (JsonMappingException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125; <span class="keyword">catch</span> (IOException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> json;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">static</span> &lt;T&gt; <span class="function">T <span class="title">str2obj</span><span class="params">(String content, Class&lt;T&gt; valueType)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">try</span> &#123;</span><br><span class="line">            <span class="keyword">return</span> mapper.readValue(content, valueType);</span><br><span class="line">        &#125; <span class="keyword">catch</span> (JsonParseException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125; <span class="keyword">catch</span> (JsonMappingException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125; <span class="keyword">catch</span> (IOException e) &#123;</span><br><span class="line">            e.printStackTrace();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">null</span>;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>同步程序的实现</strong></p>
<p>准备好实体类和工具类后，我们就可以编写同步程序来实现MySQL数据库中的数据实时同步到Solr索引库了，我们在io.mykit.canal.demo.main包中常见MykitCanalDemoSync类，代码如下所示。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> io.mykit.canal.demo.main;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> io.mykit.canal.demo.bean.Book;</span><br><span class="line"><span class="keyword">import</span> io.mykit.canal.demo.utils.BinlogValue;</span><br><span class="line"><span class="keyword">import</span> io.mykit.canal.demo.utils.CanalDataParser;</span><br><span class="line"><span class="keyword">import</span> io.mykit.canal.demo.utils.DateUtils;</span><br><span class="line"><span class="keyword">import</span> io.mykit.canal.demo.utils.InnerBinlogEntry;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.client.CanalConnector;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.client.CanalConnectors;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.CanalEntry;</span><br><span class="line"><span class="keyword">import</span> com.alibaba.otter.canal.protocol.Message;</span><br><span class="line"><span class="keyword">import</span> org.apache.solr.client.solrj.SolrServer;</span><br><span class="line"><span class="keyword">import</span> org.apache.solr.client.solrj.impl.HttpSolrServer;</span><br><span class="line"><span class="keyword">import</span> org.slf4j.Logger;</span><br><span class="line"><span class="keyword">import</span> org.slf4j.LoggerFactory;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.net.InetSocketAddress;</span><br><span class="line"><span class="keyword">import</span> java.text.ParseException;</span><br><span class="line"><span class="keyword">import</span> java.util.List;</span><br><span class="line"><span class="keyword">import</span> java.util.Map;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">SyncDataBootStart</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">static</span> Logger logger = LoggerFactory.getLogger(SyncDataBootStart.class);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> <span class="keyword">throws</span> Exception </span>&#123;</span><br><span class="line"></span><br><span class="line">        String hostname = <span class="string">&quot;192.168.175.100&quot;</span>;</span><br><span class="line">        Integer port = <span class="number">11111</span>;</span><br><span class="line">        String destination = <span class="string">&quot;example&quot;</span>;</span><br><span class="line"></span><br><span class="line">        <span class="comment">//获取CanalServer 连接</span></span><br><span class="line">        CanalConnector canalConnector = CanalConnectors.newSingleConnector(<span class="keyword">new</span> InetSocketAddress(hostname, port), destination, <span class="string">&quot;&quot;</span>, <span class="string">&quot;&quot;</span>);</span><br><span class="line"></span><br><span class="line">        <span class="comment">//连接CanalServer</span></span><br><span class="line">        canalConnector.connect();</span><br><span class="line"></span><br><span class="line">        <span class="comment">//订阅Destination</span></span><br><span class="line">        canalConnector.subscribe();</span><br><span class="line"></span><br><span class="line">        <span class="comment">//轮询拉取数据</span></span><br><span class="line">        Integer batchSize = <span class="number">5</span>*<span class="number">1024</span>;</span><br><span class="line">        <span class="keyword">while</span> (<span class="keyword">true</span>)&#123;</span><br><span class="line">            Message message = canalConnector.getWithoutAck(batchSize);</span><br><span class="line"></span><br><span class="line">            <span class="keyword">long</span> messageId = message.getId();</span><br><span class="line">            <span class="keyword">int</span> size = message.getEntries().size();</span><br><span class="line"></span><br><span class="line">            <span class="keyword">if</span>(messageId == -<span class="number">1</span> || size == <span class="number">0</span>)&#123;</span><br><span class="line">                <span class="keyword">try</span> &#123;</span><br><span class="line">                    Thread.sleep(<span class="number">1000</span>);</span><br><span class="line">                &#125; <span class="keyword">catch</span> (InterruptedException e) &#123;</span><br><span class="line">                    e.printStackTrace();</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;<span class="keyword">else</span>&#123;</span><br><span class="line">                <span class="comment">//进行数据同步</span></span><br><span class="line">                <span class="comment">//1. 解析Message对象</span></span><br><span class="line">                List&lt;InnerBinlogEntry&gt; innerBinlogEntries = CanalDataParser.convertToInnerBinlogEntry(message);</span><br><span class="line"></span><br><span class="line">                <span class="comment">//2. 将解析后的数据信息 同步到Solr的索引库中.</span></span><br><span class="line">                syncDataToSolr(innerBinlogEntries);</span><br><span class="line">            &#125;</span><br><span class="line"></span><br><span class="line">            <span class="comment">//提交确认</span></span><br><span class="line">            canalConnector.ack(messageId);</span><br><span class="line"></span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">syncDataToSolr</span><span class="params">(List&lt;InnerBinlogEntry&gt; innerBinlogEntries)</span> <span class="keyword">throws</span> Exception </span>&#123;</span><br><span class="line">        <span class="comment">//获取solr的连接</span></span><br><span class="line">        SolrServer solrServer = <span class="keyword">new</span> HttpSolrServer(<span class="string">&quot;http://192.168.175.101:8080/solr&quot;</span>);</span><br><span class="line"></span><br><span class="line">        <span class="comment">//遍历数据集合 , 根据数据集合中的数据信息, 来决定执行增加, 修改 , 删除操作 .</span></span><br><span class="line">        <span class="keyword">if</span>(innerBinlogEntries != <span class="keyword">null</span>)&#123;</span><br><span class="line">            <span class="keyword">for</span> (InnerBinlogEntry innerBinlogEntry : innerBinlogEntries) &#123;</span><br><span class="line"></span><br><span class="line">                CanalEntry.EventType eventType = innerBinlogEntry.getEventType();</span><br><span class="line"></span><br><span class="line">                <span class="comment">//如果是Insert, update , 则需要同步数据到 solr 索引库</span></span><br><span class="line">                <span class="keyword">if</span>(eventType == CanalEntry.EventType.INSERT || eventType == CanalEntry.EventType.UPDATE)&#123;</span><br><span class="line">                    List&lt;Map&lt;String, BinlogValue&gt;&gt; rows = innerBinlogEntry.getRows();</span><br><span class="line">                    <span class="keyword">if</span>(rows != <span class="keyword">null</span>)&#123;</span><br><span class="line">                        <span class="keyword">for</span> (Map&lt;String, BinlogValue&gt; row : rows) &#123;</span><br><span class="line">                            BinlogValue id = row.get(<span class="string">&quot;id&quot;</span>);</span><br><span class="line">                            BinlogValue name = row.get(<span class="string">&quot;name&quot;</span>);</span><br><span class="line">                            BinlogValue author = row.get(<span class="string">&quot;author&quot;</span>);</span><br><span class="line">                            BinlogValue publishtime = row.get(<span class="string">&quot;publishtime&quot;</span>);</span><br><span class="line">                            BinlogValue price = row.get(<span class="string">&quot;price&quot;</span>);</span><br><span class="line">                            BinlogValue publishgroup = row.get(<span class="string">&quot;publishgroup&quot;</span>);</span><br><span class="line"></span><br><span class="line">                            Book book = <span class="keyword">new</span> Book();</span><br><span class="line">                            book.setId(Integer.parseInt(id.getValue()));</span><br><span class="line">                            book.setName(name.getValue());</span><br><span class="line">                            book.setAuthor(author.getValue());</span><br><span class="line">                            book.setPrice(Double.parseDouble(price.getValue()));</span><br><span class="line">                            book.setPublishgroup(publishgroup.getValue());</span><br><span class="line">                            book.setPublishtime(DateUtils.parseDate(publishtime.getValue()));</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">                            <span class="comment">//导入数据到solr索引库</span></span><br><span class="line">                            solrServer.addBean(book);</span><br><span class="line">                            solrServer.commit();</span><br><span class="line">                        &#125;</span><br><span class="line">                    &#125;</span><br><span class="line"></span><br><span class="line">                &#125;<span class="keyword">else</span> <span class="keyword">if</span>(eventType == CanalEntry.EventType.DELETE)&#123;</span><br><span class="line">                    <span class="comment">//如果是Delete操作, 则需要删除solr索引库中的数据 .</span></span><br><span class="line">                    List&lt;Map&lt;String, BinlogValue&gt;&gt; rows = innerBinlogEntry.getRows();</span><br><span class="line">                    <span class="keyword">if</span>(rows != <span class="keyword">null</span>)&#123;</span><br><span class="line">                        <span class="keyword">for</span> (Map&lt;String, BinlogValue&gt; row : rows) &#123;</span><br><span class="line">                            BinlogValue id = row.get(<span class="string">&quot;id&quot;</span>);</span><br><span class="line"></span><br><span class="line">                            <span class="comment">//根据ID删除solr的索引库</span></span><br><span class="line">                            solrServer.deleteById(id.getValue());</span><br><span class="line">                            solrServer.commit();</span><br><span class="line">                        &#125;</span><br><span class="line">                    &#125;</span><br><span class="line"></span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<p>接下来，启动SyncDataBootStart类的main方法，监听Canal Server，而Canal Server监听MySQL binlog的日志变化，一旦MySQL的binlog日志发生变化，则SyncDataBootStart会立刻收到变更信息，并将变更信息解析成Book对象实时更新到Solr库中。如果在MySQL数据库中删除了数据，则也会实时删除Solr库中的数据。</p>
<h1 id="4-参考资源"><a href="#4-参考资源" class="headerlink" title="4 参考资源"></a>4 参考资源</h1><p><a target="_blank" rel="noopener" href="https://blog.csdn.net/u013256816/article/details/71091774">kafka数据可靠性深度解读</a></p>

    </div>

    
    
    

    <footer class="post-footer">
          <div class="post-tags">
              <a href="/blog/tags/mq/" rel="tag"># mq</a>
              <a href="/blog/tags/kafka/" rel="tag"># kafka</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-item">
                <a href="/blog/2021/01/27/RabbitMQ%E5%9F%BA%E7%A1%80%E7%BB%84%E4%BB%B6%E5%B0%81%E8%A3%85-24/" rel="prev" title="RabbitMQ基础组件封装(24)">
                  <i class="fa fa-chevron-left"></i> RabbitMQ基础组件封装(24)
                </a>
            </div>
            <div class="post-nav-item">
                <a href="/blog/2021/02/03/Java%E5%A4%9A%E7%BA%BF%E7%A8%8BJUC%E5%8E%9F%E5%AD%90%E7%B1%BB%E4%B9%8BAtomicReference/" rel="next" title="Java多线程JUC原子类之AtomicReference">
                  Java多线程JUC原子类之AtomicReference <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
    </footer>
  </article>
</div>







<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      const activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      const commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>
</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">一年春又来</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/mist/" class="theme-link" rel="noopener" target="_blank">NexT.Mist</a> 强力驱动
  </div>

    </div>
  </footer>

  
  <script src="https://cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
<script src="/blog/js/utils.js"></script><script src="/blog/js/motion.js"></script><script src="/blog/js/schemes/muse.js"></script><script src="/blog/js/next-boot.js"></script>

  
<script src="/blog/js/local-search.js"></script>






  





</body>
</html>
